[ { "path": ".coveragerc", "content": "# =========================================================================\n# COVERAGE CONFIGURATION FILE: .coveragerc\n# =========================================================================\n# LANGUAGE: Python\n# SEE ALSO:\n# * http://nedbatchelder.com/code/coverage/\n# * http://nedbatchelder.com/code/coverage/config.html\n# =========================================================================\n\n[coverage:run]\nappend = .coverage\ninclude = mailpile*\nomit = nose\nbranch = True\n#parallel = True\n\n[coverage:report]\n# Regexes for lines to exclude from consideration\nexclude_lines =\n # Have to re-enable the standard pragma\n pragma: no cover\n\n # Don't complain about missing debug-only code:\n def __repr__\n if self\\.debug\n\n # Don't complain if tests don't hit defensive assertion code:\n raise AssertionError\n raise NotImplementedError\n\n # Don't complain if non-runnable code isn't run:\n if 0:\n if False:\n if __name__ == .__main__.:\n\nignore_errors = True\n\n[coverage:html]\ndirectory = build/coverage.html\n\n[coverage:xml]\noutfile = build/coverage.xml\n\n" }, { "path": ".dockerignore", "content": "Dockerfile\nVagrantfile\n" }, { "path": ".gitignore", "content": "*.pyc\n*.pyo\n*~\n*.po\n*.debhelper\n*.iml\n*.ipr\n*.swp\n*.swo\nmp-virtualenv\n.idea\n.*deps\n.tox\nmailpile.egg-info/\nmailpile-tmp.py\nstatic/default/plugins/\nmailpile/tests/data/private-test-data.mbx\nmailpile/tests/data/tmp\nmailpile/tests/data/gpg-keyring/random_seed\nscripts/less-compiler.mk\n.DS_Store\n.vagrant\n.coverage\nghostdriver.log\nbuild/\ndist/\n.eggs/\ncover/\nbower_components/\nnode_modules/\ntesting/\nmailpile.tar.gz\nAUTHORS\nChangeLog\nshared-data/multipile/www/admin.cgi\nsetup-tmp/\n" }, { "path": ".gitmodules", "content": "[submodule \"doc\"]\n\tpath = doc\n\turl = https://github.com/pagekite/Mailpile.wiki.git\n\tbranch = master\n[submodule \"shared-data/contrib/print\"]\n\tpath = shared-data/contrib/print\n\turl = https://github.com/mailpile/Mailpile-print.git\n[submodule \"submodules/gui-o-matic\"]\n\tpath = submodules/gui-o-matic\n\turl = https://github.com/mailpile/gui-o-matic\n[submodule \"submodules/gui-o-mac-tic\"]\n\tpath = submodules/gui-o-mac-tic\n\turl = https://github.com/mailpile/gui-o-mac-tic\n" }, { "path": ".travis.yml", "content": "language: python\npython: 2.7\nenv:\n - TOX_ENV=py27\n\naddons:\n apt:\n packages:\n - gnupg\n\ninstall:\n - pip install tox coveralls\n\nscript:\n - tox -e $TOX_ENV\n\nafter_success:\n - coveralls\n" }, { "path": ".tx/config", "content": "[main]\nhost = https://www.transifex.com\n\n[mailpile.mailpilepot]\nsource_file = shared-data/locale/mailpile.pot\nsource_lang = en\ntype = PO\nfile_filter = shared-data/locale//LC_MESSAGES/mailpile.po\n" }, { "path": "AGPLv3.txt", "content": "NOTE: Please see the file COPYING.md for details on Mailpile licensing.\n\n\n GNU AFFERO GENERAL PUBLIC LICENSE\n Version 3, 19 November 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The GNU Affero General Public License is a free, copyleft license for\nsoftware and other kinds of works, specifically designed to ensure\ncooperation with the community in the case of network server software.\n\n The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works. By contrast,\nour General Public Licenses are intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n Developers that use our General Public Licenses protect your rights\nwith two steps: (1) assert copyright on the software, and (2) offer\nyou this License which gives you legal permission to copy, distribute\nand/or modify the software.\n\n A secondary benefit of defending all users' freedom is that\nimprovements made in alternate versions of the program, if they\nreceive widespread use, become available for other developers to\nincorporate. Many developers of free software are heartened and\nencouraged by the resulting cooperation. However, in the case of\nsoftware used on network servers, this result may fail to come about.\nThe GNU General Public License permits making a modified version and\nletting the public access it on a server without ever releasing its\nsource code to the public.\n\n The GNU Affero General Public License is designed specifically to\nensure that, in such cases, the modified source code becomes available\nto the community. It requires the operator of a network server to\nprovide the source code of the modified version running there to the\nusers of that server. Therefore, public use of a modified version, on\na publicly accessible server, gives the public access to the source\ncode of the modified version.\n\n An older license, called the Affero General Public License and\npublished by Affero, was designed to accomplish similar goals. This is\na different license, not a version of the Affero GPL, but Affero has\nreleased a new version of the Affero GPL which permits relicensing under\nthis license.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n TERMS AND CONDITIONS\n\n 0. Definitions.\n\n \"This License\" refers to version 3 of the GNU Affero General Public License.\n\n \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n \"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License. If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n 1. Source Code.\n\n The \"source code\" for a work means the preferred form of the work\nfor making modifications to it. \"Object code\" means any non-source\nform of a work.\n\n A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities. However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work. For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n The Corresponding Source for a work in source code form is that\nsame work.\n\n 2. Basic Permissions.\n\n All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force. You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright. Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n Conveying under any other circumstances is permitted solely under\nthe conditions stated below. Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n 4. Conveying Verbatim Copies.\n\n You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n 5. Conveying Modified Source Versions.\n\n You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n a) The work must carry prominent notices stating that you modified\n it, and giving a relevant date.\n\n b) The work must carry prominent notices stating that it is\n released under this License and any conditions added under section\n 7. This requirement modifies the requirement in section 4 to\n \"keep intact all notices\".\n\n c) You must license the entire work, as a whole, under this\n License to anyone who comes into possession of a copy. This\n License will therefore apply, along with any applicable section 7\n additional terms, to the whole of the work, and all its parts,\n regardless of how they are packaged. This License gives no\n permission to license the work in any other way, but it does not\n invalidate such permission if you have separately received it.\n\n d) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your\n work need not make them do so.\n\n A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit. Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n 6. Conveying Non-Source Forms.\n\n You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n a) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium\n customarily used for software interchange.\n\n b) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a\n written offer, valid for at least three years and valid for as\n long as you offer spare parts or customer support for that product\n model, to give anyone who possesses the object code either (1) a\n copy of the Corresponding Source for all the software in the\n product that is covered by this License, on a durable physical\n medium customarily used for software interchange, for a price no\n more than your reasonable cost of physically performing this\n conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n\n c) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This\n alternative is allowed only occasionally and noncommercially, and\n only if you received the object code with such an offer, in accord\n with subsection 6b.\n\n d) Convey the object code by offering access from a designated\n place (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to\n copy the object code is a network server, the Corresponding Source\n may be on a different server (operated by you or a third party)\n that supports equivalent copying facilities, provided you maintain\n clear directions next to the object code saying where to find the\n Corresponding Source. Regardless of what server hosts the\n Corresponding Source, you remain obligated to ensure that it is\n available for as long as needed to satisfy these requirements.\n\n e) Convey the object code using peer-to-peer transmission, provided\n you inform other peers where the object code and Corresponding\n Source of the work are being offered to the general public at no\n charge under subsection 6d.\n\n A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling. In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage. For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product. A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source. The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information. But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n 7. Additional Terms.\n\n \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit. (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.) You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n a) Disclaiming warranty or limiting liability differently from the\n terms of sections 15 and 16 of this License; or\n\n b) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal\n Notices displayed by works containing it; or\n\n c) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n\n d) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n\n e) Declining to grant rights under trademark law for use of some\n trade names, trademarks, or service marks; or\n\n f) Requiring indemnification of licensors and authors of that\n material by anyone who conveys the material (or modified versions of\n it) with contractual assumptions of liability to the recipient, for\n any liability that these contractual assumptions directly impose on\n those licensors and authors.\n\n All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n 8. Termination.\n\n You may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n 9. Acceptance Not Required for Having Copies.\n\n You are not required to accept this License in order to receive or\nrun a copy of the Program. Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance. However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work. These actions infringe copyright if you do\nnot accept this License. Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n 10. Automatic Licensing of Downstream Recipients.\n\n Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\n An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n 11. Patents.\n\n A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The\nwork thus licensed is called the contributor's \"contributor version\".\n\n A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version. For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License. You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n 12. No Surrender of Others' Freedom.\n\n If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all. For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n 13. Remote Network Interaction; Use with the GNU General Public License.\n\n Notwithstanding any other provision of this License, if you modify the\nProgram, your modified version must prominently offer all users\ninteracting with it remotely through a computer network (if your version\nsupports such interaction) an opportunity to receive the Corresponding\nSource of your version by providing access to the Corresponding Source\nfrom a network server at no charge, through some standard or customary\nmeans of facilitating copying of software. This Corresponding Source\nshall include the Corresponding Source for any work covered by version 3\nof the GNU General Public License that is incorporated pursuant to the\nfollowing paragraph.\n\n Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU General Public License into a single\ncombined work, and to convey the resulting work. The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the work with which it is combined will remain governed by version\n3 of the GNU General Public License.\n\n 14. Revised Versions of this License.\n\n The Free Software Foundation may publish revised and/or new versions of\nthe GNU Affero General Public License from time to time. Such new versions\nwill be similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nProgram specifies that a certain numbered version of the GNU Affero General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation. If the Program does not specify a version number of the\nGNU Affero General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n If the Program specifies that a proxy can decide which future\nversions of the GNU Affero General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n Later license versions may give you additional or different\npermissions. However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n 15. Disclaimer of Warranty.\n\n THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n 16. Limitation of Liability.\n\n IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n 17. Interpretation of Sections 15 and 16.\n\n If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU Affero General Public License as published by\n the Free Software Foundation, either version 3 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU Affero General Public License for more details.\n\n You should have received a copy of the GNU Affero General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\n If your software can interact with users remotely through a computer\nnetwork, you should also make sure that it provides a way for users to\nget its source. For example, if your program is a web application, its\ninterface could display a \"Source\" link that leads users to an archive\nof the code. There are many ways you could offer source, and different\nsolutions will be better for different programs; see section 13 for the\nspecific requirements.\n\n You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU AGPL, see\n.\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# The Mailpile Code of Conduct\n\nThe Mailpile project depends on its community of volunteers, supporters and of\ncourse users. We want people to feel welcome and respected. To this end,\nproject maintainers have plegded to adhere to and enforce the Contrbitor\nCovenant Code of Conduct, in their Mailpile related activities, both online and\nin person.\n\nWe respectfully ask that other contributors and participants in our community\ndo so as well.\n\nThank you for your understanding, and welcome!\n\n\n# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, gender identity and expression, level of experience,\nnationality, personal appearance, race, religion, or sexual identity and\norientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at . All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at [http://contributor-covenant.org/version/1/4][version]\n\n[homepage]: http://contributor-covenant.org\n[version]: http://contributor-covenant.org/version/1/4/\n" }, { "path": "CONTRIBUTING.md", "content": "## How to contribute to Mailpile\n\nFirst of all: Thank you! :heart:\n\n\n#### Further Reading\n\nSecond of all, please adhere to our Code of Conduct when you participate\nin our community. Be kind, be respectful. [The full Code of Conduct can\nbe found here](CODE_OF_CONDUCT.md).\n\nPlease also be sure you are comfortable with [our license](COPYING.md)\nbefore contributing any code to Mailpile.\n\nNote that Mailpile is a slightly unsual app; the design is both political\nand opinionated. If you are confused or unsure whether something is a bug or a\nfeature, [our developer FAQ might have answers](DEV_FAQ.md)! Please read it\nbefore filing issues about design choices or attempting to reorganize the code\nin any major way. The FAQ also contains a quick introduction Mailpile internals\nand debugging techniques.\n\nAlso, have you seen [the main Mailpile website](https://www.mailpile.is/) and\n[the community Discourse](https://community.mailpile.is/categories)?\n\n\n#### Are you new to Mailpile and/or FOSS?\n\n* Welcome! The first step is probably to get Mailpile up and running, play with it, get familiar.\n\n* Read the links above, and [check out our website](https://www.mailpile.is/) if you haven't already.\n\n* Not sure what to work on? Find inspiration in one of our [Low Hanging Fruit](https://github.com/mailpile/Mailpile/issues?q=is%3Aissue+is%3Aopen+label%3A%22Low+Hanging+Fruit%22) issues!\n\n* Otherwise, read on...\n\n\n#### Did you find a bug?\n\n* **Ensure the bug was not already reported** by searching on GitHub\n under [Issues](https://github.com/mailpile/Mailpile/issues).\n\n* If you're unable to find an open issue addressing the problem,\n [open a new one](https://github.com/mailpile/Mailpile/issues/new). Be\n sure to include a **title and clear description**, as much relevant\n information as possible.\n\n* Remember, we cannot read your mind or see your screen, so you'll\n probably need to answer all of these:\n\n 1. What were you doing?\n 2. What did you expect would happen?\n 3. What actually happened?\n 4. Operating system? GnuPG version? Mailpile version?\n\n* Reproducability is key. If you cannot reliably trigger the bug or\n cannot describe how to do so, then unfortunately it's less likely that\n the Mailpile team will be able to do anything about it. It may still\n be useful to file a report in case others are having the same issue, but\n bugs that can be reproduced will in general get fixed much faster!\n\n* The [Developer FAQ](DEV_FAQ.md) has a section on debugging techniques.\n\n* If it's not bug, but you still need help: [visit our Discourse support\n forum](https://community.mailpile.is/c/support).\n\n\n#### Did you write a patch that fixes a bug?\n\n* Open a new GitHub pull request with the patch.\n\n* Ensure the PR description clearly describes the problem and solution.\n Include the relevant issue number if applicable.\n\n\n#### Do you plan write a patch that adds a feature?\n\n* Please be sure you have read the [Developer FAQ](DEV_FAQ.md) to ensure\n your feature is compatible with our high-level goals and design decisions.\n\n* If you are unsure or would like some guidance, join the #mailpile\n channel on IRC (Freenode) and discuss your plans there.\n\n* Open a new GitHub pull request with the patch.\n\n* Ensure the PR description clearly describes what your feature does.\n Include the relevant issue number if applicable.\n\n\n#### Did you fix whitespace, format code, or make a purely cosmetic patch?\n\nChanges that are cosmetic in nature and do not add anything substantial to the\nstability, functionality, or testability of Mailpile will generally not be\naccepted.\n\nAll patches to Mailpile are reviewed by a human and our resources (time,\npeople) are very limited. Cosmetic patches are no easier to review than other\npatches and we would rather focus our efforts on functional improvements to the\nsoftware.\n\n\n#### Do you want help other users?\n\nOur [community Discourse](https://community.mailpile.is/categories) has a\nsupport forum and needs friendly people to help out and steer conversations\nin friendly and productive directions. Please join the conversation!\n\n\n#### Do you want to translate Mailpile to your language?\n\nPlease feel free to join our translation community\n[on Transifex](https://www.transifex.com/otf/mailpile/).\n\n\n#### Do you plan to write documentation?\n\n* Please feel free to contribute documentation to\n [our wiki](https://github.com/mailpile/Mailpile/wiki).\n\n* If you would like some guidance, join the #mailpile channel on IRC\n (Freenode) or file an issue with your questions.\n\n* If you added a new page or made major major changes to an existing page,\n please file [a new issue requesting a review](https://github.com/mailpile/Mailpile/issues/new)\n when you are done. Be sure to include a link to the wiki page!\n\n\n#### Credits\n\nThis document borrows some language and structure from the [Ruby on Rails\ncontributor guidelines](https://raw.githubusercontent.com/rails/rails/master/CONTRIBUTING.md).\n\n" }, { "path": "COPYING.md", "content": "# Mailpile - a program for doing stuff with e-mail\n\nCopyright (C) 2011-2015, Bjarni R. Einarsson, Mailpile ehf and friends.\n\nThis program is free software: you can redistribute it and/or modify it\nunder the terms of the GNU Affero General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or (at\nyour option) any later version.\n\nThis program is distributed in the hope that it will be useful, but\nWITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero\nGeneral Public License more details.\n\nYou should have received a copy of the GNU Affero General Public License\nalong with this program. If not, see .\n\n\n## What happened to the Apache license?\n\nThis software used to be dual-licensed under AGPLv3 and the Apache\nLicense 2.0. If you feel an uncontrollable urge to fork and carry on\nwithout the AGPL, the last dual-licensed code point is tagged as\n\"Gunsmoke--TheLastApacheTag\". Have fun!\n\nFor more details on our licensing choices, see these blog posts:\n\n* \n* \n* \n\nThanks!\n" }, { "path": "DEV_FAQ.md", "content": "## Mailpile Developer FAQ\n\nThis document contain a collection of frequently asked questions (with\nanswers) about Mailpile development. Please familiarize yourself with\nthe contents before attempting any deep hacking on Mailpile.\n\nYou don't have to agree with all of our priorities to take part or make\nuse of Mailpile, but we do feel it helps if most of the community is\nrowing in roughly the same general direction!\n\nNote: If you are just looking for debugging tips and tricks, you can skip\nto the end.\n\n\n### Why Mailpile?\n\nThe long-term goal of Mailpile is to help *non-technical people* become\n*more independent* and *more private* online, in particular when it\ncomes to e-mail.\n\nBy **more independent**, we mean people should be in control of their\ne-mail and the software used to manage it. This is why Mailpile is Free\nSoftware, and this is why we *don't* promote Mailpile as a tool for\nbuilding \"cloud services\".\n\nBy **more private**, we mean people should have more control over who has\naccess to their e-mail, when and how. This is why Mailpile tries to make\ne-mail encryption more accessible, and this is why Mailpile tries to\nmake it convenient *and secure* for people to store their e-mail on their\nown devices.\n\nOur focus on **non-technical people** implies, amongst other things, that\nwe cannot exclude people who are using non-free platforms such as\nMicrosoft Windows or Mac OS X, and we cannot require our users learn a\nradically new, unfamiliar user interface or understand highly technical\nconcepts such as public key cryptography.\n\n\n### Are All of These Decisions Awesome?\n\nNo. Some of these decisions were almost certainly mistakes. But that's\npart of innovating, you try new things and see how they go!\n\nSo we kindly request that our contributors and collaboratores refrain\nfrom picking arguments with us over these decisions. Making forward\nprogress is much more fun than rehashing the past over and over again.\n\n\n### Why Do You Reinvent So Many Wheels?\n\nUsually, the answer is one or more of the following:\n\n * We didn't know a solution already existed\n * We evaluated solutions X, Y and Z, but didn't like them\n * We felt the problem was simple enough to Just Do It\n\nIn general, we are reluctant to take on new external dependencies unless\nthey are stable and widely available: if they exist in pre-packaged form\nfor Windows, Mac OS X and the most common Linux distributions, they are\nprobably fine!\n\nIf not, the benefits need to outweigh the added complexity posed for our\ncross-platform packaging efforts.\n\nFor this reason we also tend to prefer pure-Python (or Javascript, for\nthe UI) libraries over native code, which also avoids certain classes of\nbugs and security vulnerabilities which are simply not present in a\nmemory-safe, managed language.\n\nWe have broken this rule more often than we'd like in our user-interface\ncode, due to cultural differences between web developers and Python\nfolks. *We would like to gradually reduce our front-end dependencies.*\n\n\n### Why a Search Engine?\n\nMailpile started as an experimental search engine, a hobby project.\nEverything else came later.\n\n*OK, so why not replace it with a standard component?*\n\nBecause it works! Replacing it at this point would be *more work, not less*.\n\nAlso, we are unaware of any \"off the shelf\" search engines that let us\nencrypt their data stores and we feel encrypting the search index is an\nabsolute requirement since we by default allow the user to search inside\nencrypted messages.\n\n*I see. Why not remove it, to simplify the code?*\n\nThe entire app is built around the search metaphor, much like Google's\nGMail. This is a fundamentally different way to build an e-mail client,\nfrom the traditional \"messages and mailboxes\" model.\n\nThe search engine also makes it easy for Mailpile to do some pretty cool\nthings. For example:\n\n * Mailpile can evaluate the trustworthiness of a message by asking the\n search engine about the past behaviour of the sender\n * Mailpile can postpone processing of things like attached PGP keys\n until it actually intends to use the key: the search engine makes the\n keys easy to find later on\n\n\n### Why doesn't Mailpile have a Native User Interface?\n\nFor now, because the team is very small and we would like to reach users\non many different platforms, we have bet entirely on the web as our\nprimary user interface:\n\n * This decision allows us to target any operating system for which\n Python 2.7 has been made available.\n\n * It also allows us to get help from the enormous community of web\n developers; this is a much larger talent pool than the pool of\n developers that know how to write native apps.\n\n * Last but not least, the web interface is key to our plan to allow\n users to access their Mailpile remotely over the network. Remote\n access is critical if we want to get people to store their e-mail\n on their own devices, because most people read their mail on\n multiple devices (laptop, tablet, mobile, work computer, etc).\n\nThat said: we do want to have a minimal native user interface, on all\nthe major desktop operating systems. [That is what the gui-o-matic\nspin-off project is about](https://github.com/mailpile/gui-o-matic).\n\nNative mobile apps on Android and/or iOS would also be nice to have!\n\n\n### How Does Mailpile Handle Security?\n\nThis is a huge topic! Please consult our [Security\nRoadmap](https://github.com/mailpile/Mailpile/wiki/Security-roadmap).\n\nSecurity is complex and means different things to different people.\n\nThe most controversial questions relating to security, have to do with\nmass surveillance and law enforcement. Mailpile's stance is that we\nbelieve that people have an innate right to privacy we believe that mass\nsurveillance is *wrong*. If the government wants to read your e-mail,\nwe feel they should present you with a search warrant.\n\nThwarting other adversaries (criminals, jealous partners, etc.) is also\nvery much something we care about, but is probably less divisive.\n\n\n### Why GnuPG? Why not GPGME? Why not PGPy?\n\nGnuPG is mature and stable. Although the user interfaces may leave\nsomething to be desired, it has a rich ecosystem of powerful tools built\naround it and a wealth of documentation and support to be found online.\nIf we didn't use GnuPG, we would have to reinvent a lot of wheels that\naren't central to Mailpile's mission.\n\nOur issue tracker contains [further discussions on the topic of why\nGnuPG and not something else, such as\nPGPy](https://github.com/mailpile/Mailpile/issues/1743).\n\n[Use of GPGME is also being\ndiscussed](https://github.com/mailpile/Mailpile/issues/1742). Currently,\nwe don't feel GPGME provides enough benefit to justify the additional\ndependency and the additional (hypothetical) risk posed by solving the\nGnuPG integration problem with a large amount of code written in C (as\nopposed to Python, which is memory-safe and less likely to contain\ncertain classes of security vulnerabilities).\n\n\n### Why not Python 3?\n\nWe depend on some libraries - spambayes in particular - which were not\navailable for Python 3 when we started this project.\n\nWe don't think Python 2 is going away in the near future.\n\n[There is a Github issue discussing\nthis.](https://github.com/mailpile/Mailpile/issues/160)\n\n\n### Why Not Django? Or Flask?\n\nReasons!\n\nOur way may be unusual, but it's kinda awesome once you get used to it\nand it wasn't obvious to us how we could get this kind of behaviour from\none of the standard frameworks.\n\nPlease read the next section for details.\n\n\n### How Does Mailpile's Web UI Work?\n\nThe text-based command-line interface is an important part of Mailpile's\nuser interface. Our home-brewed framework allows us to generate web API\nend-points, text commands and command-line arguments *at the same time*.\n\nOur internal framework also has the concept of commands supporting\nmultiple output formats; so the same API endpoint can generate text,\ntemplated HTML, JSON and XML-RPC interfaces with relatively little\nadditional code. Some endpoints also generate XML, CSV, CSS or\nJavascript.\n\nYou can try this yourself, simply by editing the URL in your browser:\n\n # The default, rendered as HTML\n http://localhost:33411/in/inbox/\n http://localhost:33411/in/inbox/as.html\n http://localhost:33411/search/?q=in:inbox\n\n # Same thing, as JSON\n http://localhost:33411/in/inbox/as.json\n http://localhost:33411/api/0/search/?q=in:inbox\n\n # Same thing, as text\n http://localhost:33411/in/inbox/as.text\n http://localhost:33411/api/0/search/as.text?q=in:inbox\n\nThe filename part of the URL is used to select output formats. All\nendpoints support `as.json`, most support HTML and/or text.\n\nThe HTML output of each command is generated using Jinja2 templates\nthat are found in `shared-data/default-theme/html/...`. The directory\nstructure generally matches the URL paths seen in the browser, with\nthe main template for each command named `index.html`.\n\nAlternate templates for the same API endpoint can have other names, for\nexample a template named `.../html/search/social.html` would be\naccessible using URLs like so:\n\n http://localhost:33411/in/inbox/social.html\n http://localhost:33411/search/social.html?q=from:person@foo.com\n\n\n### Can I Develop Plugins For Mailpile?\n\nSort-of!\n\nInternally, the app is quite modular and there are methods which allow\ncode to register classes or functions that perform various functions.\n\nHowever, the plugin API is not considered stable, it is incomplete and\nit is not very well documented. It may also not be a very nice API, and\nwe rather expect it to develop and change rapidly post-1.0.\n\nIf you are interested in Mailpile's plugin APIs, take a look in\n`shared-data/contrib/` for some examples of \"external plugins\" and\n`mailpile/plugins/` for \"internal plugins\".\n\n\n### How Do I Debug Mailpile?\n\nDevelopers should learn to use the Mailpile CLI. The `mailpile>` prompt is\nwhere all of the low-level magic happens. Future versions of Mailpile will\nexpose this functionality to the web interface itself, but for now you will\nneed to use your shell.\n\nPossibly the most important command for Mailpile hackers, is to know\nhow to enable debugging. An example:\n\n # Enable verbose debugging of HTTP requests and GnuPG integration\n # Note: HTTP debugging disables all sorts of internal caches!\n mailpile> set sys.debug = log http gnupg\n ...\n\nMany other subsystems can have debugging enabled. At the time of writing,\nthe `sys.debug` can include the following terms to make various parts of\nthe app more verbose:\n\n log http compose cryptostate autotag rescan keywords cache connbroker\n vcard pop3 gnupg keylookup imap sources jinja timing sendmail httpdata\n\nThere are also a few other ways to examine the app state:\n\n # Watch logging and debug messages fly by\n mailpile> eventlog/watch\n ...\n [CTRL+C]\n ...\n\n # Examine event log (piped through less)\n mailpile> pipe less eventlog\n ...\n mailpile> pipe less eventlog incomplete\n ...\n\n # Get an overview of what threads are running and what they are doing\n mailpile> ps\n ...\n\nLow-level changes and exploration of the configuration are also best done\nfrom the CLI:\n\n # Explore the configuration; see also mailpile/config/defaults.py\n mailpile> print -short sys\n ...\n mailpile> print -flat sources\n ...\n mailpile> print -secrets secrets\n ...\n \n # Change things (dangerous)\n mailpile> set sys.gpg_binary = /bin/false\n ...\n\n # Reset something to its default setting\n mailpile> unset sys.gpg_binary\n ...\n\nThere is also a help command, and you can use tab completion to try and\n\"guess\" what commands exist.\n\n mailpile> help\n ...\n mailpile> help tags\n ...\n\nFinally, the app ships with a `hacks` plugin which is disabled by default.\nIf you load it, that will add a few more low-level commands, including an\nembedded Python interpretor:\n\n mailpile> plugins/load hacks\n ...\n\n mailpile> hacks/pycli\n ...\n\nThere's sure to be more; please feel free to file a pull request against\nthis document to add your favourite tricks or clarify these.\n" }, { "path": "Dockerfile", "content": "FROM debian:stretch-slim\n\nENV GID 33411\nENV UID 33411\n\nRUN apt-get update && \\\n apt-get install -y curl apt-transport-https gnupg && \\\n curl -s https://packages.mailpile.is/deb/key.asc | apt-key add - && \\\n echo \"deb https://packages.mailpile.is/deb release main\" | tee /etc/apt/sources.list.d/000-mailpile.list && \\\n apt-get update && \\\n apt-get install -y mailpile && \\\n # TODO Enable apache for multi users\n # apt-get install -y mailpile-apache2\n update-rc.d tor defaults && \\\n service tor start && \\\n groupadd -g $GID mailpile && \\\n useradd -u $UID -g $GID -m mailpile && \\\n su - mailpile -c 'mailpile setup' && \\\n apt-get clean\n\nWORKDIR /home/mailpile\nUSER mailpile\n\nVOLUME /home/mailpile/.local/share/Mailpile\nEXPOSE 33411\n\nCMD mailpile --www=0.0.0.0:33411/ --wait\n" }, { "path": "Dockerfile.dev", "content": "FROM mailpile\n\n# Install C compiler for python deps w/ native extensions\nRUN apk --no-cache add \\\n gcc \\\n libc-dev \\\n python-dev \\\n shadow\n\n# Workaround: Setting mailpile users uid to 1000 to have write permissions\n# from the docker host the to the shared volumes /Mailpile and /mailpile-data.\n# Mounted volumes seem to be configured w/ uid/guid = 1000.\n# Learn more here:\n# - https://github.com/docker/docker/issues/7198\n# - https://denibertovic.com/posts/handling-permissions-with-docker-volumes/\nRUN usermod -u 1000 mailpile\n\nRUN pip install -r requirements-dev.txt\n\nRUN chmod +x /entrypoint.sh\n\nCMD [\"./mp\"]\n" }, { "path": "Gruntfile.js", "content": "module.exports = function(grunt) {\n\n grunt.registerTask('watch', [ 'watch' ]);\n\n grunt.initConfig({\n concat: {\n js: {\n options: {\n separator: ';'\n },\n src: [\n 'bower_components/jquery/dist/jquery.min.js',\n 'bower_components/underscore/underscore-min.js',\n 'bower_components/jquery-timer/jquery.timer.js',\n 'bower_components/autosize/dist/autosize.js',\n 'bower_components/mousetrap/mousetrap.js',\n 'shared-data/default-theme/js/mousetrap.global.bind.js',\n 'bower_components/jquery.ui/ui/jquery.ui.core.js',\n 'bower_components/jquery.ui/ui/jquery.ui.widget.js',\n 'bower_components/jquery.ui/ui/jquery.ui.mouse.js',\n 'bower_components/jquery.ui/ui/jquery.ui.draggable.js',\n 'bower_components/jquery.ui/ui/jquery.ui.droppable.js',\n 'bower_components/jquery.ui/ui/jquery.ui.sortable.js',\n 'bower_components/jqueryui-touch-punch/jquery.ui.touch-punch.js',\n 'bower_components/qtip2/basic/jquery.qtip.min.js',\n 'bower_components/jquery-slugify/dist/slugify.js',\n 'bower_components/typeahead.js/dist/typeahead.jquery.js',\n 'bower_components/bootstrap/js/dropdown.js',\n 'bower_components/bootstrap/js/modal.js',\n 'bower_components/favico.js/favico.js',\n 'bower_components/select2/select2.min.js',\n 'bower_components/moxie/bin/js/moxie.min.js',\n 'bower_components/plupload/js/plupload.min.js',\n 'bower_components/dompurify/dist/purify.min.js'\n ],\n dest: 'shared-data/default-theme/js/libraries.min.js'\n }\n },\n uglify: {\n options: {\n mangle: false\n },\n js: {\n files: {\n 'shared-data/default-theme/js/libraries.min.js': ['shared-data/default-theme/js/libraries.min.js']\n }\n }\n },\n less: {\n options: {\n cleancss: true\n },\n style: {\n files: {\n \"shared-data/default-theme/css/default.css\": \"shared-data/default-theme/less/default.less\"\n }\n }\n },\n watch: {\n js: {\n files: ['shared-data/default-theme/js/*.js'],\n tasks: ['concat:js', 'uglify:js'],\n options: {\n livereload: true,\n }\n },\n css: {\n files: [\n 'shared-data/default-theme/less/config.less',\n 'shared-data/default-theme/less/default.less',\n 'shared-data/default-theme/less/app/*.less',\n 'shared-data/default-theme/less/libraries/*.less'\n ],\n tasks: ['less:style'],\n options: {\n livereload: true,\n }\n }\n }\n });\n\n grunt.loadNpmTasks('grunt-contrib-concat');\n grunt.loadNpmTasks('grunt-contrib-uglify');\n grunt.loadNpmTasks('grunt-contrib-less');\n grunt.loadNpmTasks('grunt-contrib-watch');\n\n};\n" }, { "path": "MANIFEST.in", "content": "recursive-exclude locale *\nrecursive-exclude scripts *\nrecursive-exclude testing *\nrecursive-include mailpile/tests/data *\nrecursive-exclude mailpile/tests/data *~\nrecursive-include mailpile/locale *.mo *.po *.pot\nrecursive-include mailpile/plugins *\ninclude Makefile\ninclude README.md\ninclude AGPLv3.txt\ninclude LICENSE-2.0.txt\ninclude requirements.txt\ninclude test-requirements.txt\ninclude requirements-dev.txt\ninclude install_hooks.py\ninclude scripts/compile-messages.sh\ninclude scripts/make-messages.sh\ninclude scripts/mailpile-test.py\ninclude scripts/test-sendmail.sh\ninclude scripts/setup-test.sh\nrecursive-include shared-data *\n" }, { "path": "Makefile", "content": "# Recipes for stuff\nexport PYTHONPATH := .\n\nhelp:\n\t@echo \"\"\n\t@echo \"BUILD\"\n\t@echo \" dpkg\"\n\t@echo \" Create a debian package of this service (in a Docker \"\n\t@echo \" container).\"\n\t@echo \"\"\n\n\nall:\tsubmodules alltests docs web compilemessages transifex\n\ndev:\n\t@echo export PYTHONPATH=`pwd`\n\narch-dev:\n\tsudo pacman -Syu --needed community/python2-pillow extra/python2-lxml community/python2-jinja \\\n\t community/python2-pep8 extra/python2-nose community/phantomjs \\\n\t extra/python2-pip community/python2-mock \\\n\t extra/ruby community/npm community/spambayes\n\tTMPDIR=`mktemp -d /tmp/aur.XXXXXXXXXX`; \\\n\tcd $$TMPDIR; \\\n\tpacman -Qs '^yuicompressor$$' > /dev/null; \\\n\tif [ $$? -ne 0 ]; then \\\n\t sudo pacman -S --needed core/base-devel; \\\n\t curl -s https://aur.archlinux.org/cgit/aur.git/snapshot/yuicompressor.tar.gz | tar xzv; \\\n\t cd yuicompressor; \\\n\t makepkg -si; \\\n\t cd $$TMPDIR; \\\n\tfi; \\\n\tcd /tmp; \\\n\trm -rf $$TMPDIR\n\tsudo pip2 install 'selenium>=2.40.0'\n\twhich lessc >/dev/null || sudo gem install therubyracer less\n\twhich bower >/dev/null || sudo npm install -g bower\n\twhich uglify >/dev/null || sudo npm install -g uglify\n\nfedora-dev:\n\tsudo yum install python-imaging python-lxml python-jinja2 python-pep8 \\\n\t ruby ruby-devel python-yui python-nose spambayes \\\n\t phantomjs python-pip python-mock npm\n\tsudo yum install rubygems; \\\n\tsudo yum install python-pgpdump || pip install pgpdump\n\tsudo pip install 'selenium>=2.40.0'\n\twhich lessc >/dev/null || sudo gem install therubyracer less\n\twhich bower >/dev/null || sudo npm install -g bower\n\twhich uglify >/dev/null || sudo npm install -g uglify\n\ndebian-dev:\n\tsudo apt-get install python-imaging python-lxml python-jinja2 pep8 \\\n\t ruby-dev yui-compressor python-nose spambayes \\\n\t python-pip python-mock python-selenium \\\n\t\t\t\t\t\t rubygems-integration\n\tdpkg -l|grep -qP ' nodejs .*nodesource' || sudo apt install npm\n\tsudo apt-get install python-pgpdump || pip install pgpdump\n\twhich phantomjs >/dev/null || sudo apt-get install phantomjs || sudo npm install -g phantomjs\n\twhich lessc >/dev/null || sudo gem install therubyracer less\n\twhich bower >/dev/null || sudo npm install -g bower\n\twhich uglify >/dev/null || sudo npm install -g uglify\n\n\nsubmodules:\n\tgit submodule update --remote\n\ndocs: submodules\n\t@python2.7 mailpile/urlmap.py |grep -v ^FIXME: >doc/URLS.md\n\t@ls -l doc/URLS.md\n\t@python2.7 mailpile/defaults.py |grep -v -e ^FIXME -e ';timestamp' \\\n >doc/defaults.cfg\n\t@ls -l doc/defaults.cfg\n\nweb: less js\n\t@true\n\nalltests: clean pytests\n\t@chmod go-rwx mailpile/tests/data/gpg-keyring\n\t@DISPLAY= nosetests\n\t@DISPLAY= python2.7 scripts/mailpile-test.py || true\n\t@git checkout mailpile/tests/data/\n\npytests:\n\t@echo -n 'security ' && python2.7 mailpile/security.py\n\t@echo -n 'urlmap ' && python2.7 mailpile/urlmap.py -nomap\n\t@echo -n 'search ' && python2.7 mailpile/search.py\n\t@echo -n 'mailboxes.mbox ' && python2.7 mailpile/mailboxes/mbox.py\n\t@echo -n 'mailutils.safe ' && python2.7 mailpile/mailutils/safe.py\n\t@echo -n 'mailutils.addrs ' && python2.7 mailpile/mailutils/addresses.py\n\t@echo -n 'mailutils.emails ' && python2.7 mailpile/mailutils/emails.py\n\t@echo -n 'config/base ' && python2.7 mailpile/config/base.py\n\t@echo -n 'config/validators' && python2.7 mailpile/config/validators.py\n\t@echo -n 'config/manager ' && python2.7 mailpile/config/manager.py\n\t@echo -n 'conn_brokers ' && python2.7 mailpile/conn_brokers.py\n\t@echo -n 'crypto/autocrypt ' && python2.7 mailpile/crypto/autocrypt.py\n\t@echo -n 'plug...autocrypt ' && python2.7 mailpile/plugins/crypto_autocrypt.py\n\t@echo -n 'crypto/mime ' && python2.7 mailpile/crypto/mime.py\n\t@echo -n 'index.base ' && python2.7 mailpile/index/base.py\n\t@echo -n 'index.msginfo ' && python2.7 mailpile/index/msginfo.py\n\t@echo -n 'index.mailboxes ' && python2.7 mailpile/index/mailboxes.py\n\t@echo -n 'index.search ' && python2.7 mailpile/index/search.py\n\t@echo -n 'util ' && python2.7 mailpile/util.py\n\t@echo -n 'vcard ' && python2.7 mailpile/vcard.py\n\t@echo -n 'workers ' && python2.7 mailpile/workers.py\n\t@echo -n 'packing ' && python2.7 mailpile/packing.py\n\t@echo -n 'mailboxes/pop3 ' && python2.7 mailpile/mailboxes/pop3.py\n\t@echo -n 'mail_source/imap ' && python2.7 mailpile/mail_source/imap.py\n\t@echo -n 'crypto/aes_utils ' && python2.7 mailpile/crypto/aes_utils.py\n\t@echo 'spambayes... ' && python2.7 mailpile/spambayes/Tester.py\n\t@echo 'crypto/streamer...' && python2.7 mailpile/crypto/streamer.py\n\t@echo\n\nclean:\n\t@rm -f `find . -name \\\\*.pyc` \\\n\t `find . -name \\\\*.pyo` \\\n\t `find . -name \\\\*.mo` \\\n\t mailpile-tmp.py mailpile.py \\\n\t ChangeLog AUTHORS \\\n\t .appver MANIFEST .SELF .*deps \\\n dist/*.tar.gz dist/*.deb dist/*.rpm \\\n\t scripts/less-compiler.mk ghostdriver.log\n\t@rm -rf *.egg-info build/ \\\n mailpile/tests/data/tmp/ testing/tmp/\n\t@rm -f shared-data/multipile/www/admin.cgi\n\nmrproper: clean\n\t@rm -rf shared-data/locale/?? shared-data/locale/??[_@]*\n\t@rm -rf bower_components/ shared-data/locale/mailpile.pot\n\t@rm -rf mp-virtualenv/\n\tgit reset --hard && git clean -dfx\n\nsdist: clean\n\t@python setup.py sdist\n\n#bdist-prep: compilemessages web -- FIXME: Make building web assets work!\nbdist-prep: compilemessages\n\t@true\n\nbdist:\n\t@python setup.py bdist_wheel\n\nvirtualenv: mp-virtualenv/bin/activate\nvirtualenv-dev: mp-virtualenv/bin/.dev\n\nmp-virtualenv/bin/activate:\n\tvirtualenv -p python2.7 --system-site-packages mp-virtualenv\n\tbash -c 'source mp-virtualenv/bin/activate && pip install -r requirements.txt && python setup.py install'\n\t@rm -rf mp-virtualenv/bin/.dev\n\t@echo\n\t@echo NOTE: If you want to test/develop with GnuPG 2.1, you might\n\t@echo want to activate the virtualenv and then run this script\n\t@echo to build GnuPG 2.1: ./scripts/add-gpgme-and-gnupg-to-venv\n\t@echo\n\nmp-virtualenv/bin/.dev: virtualenv\n\trm -rf mp-virtualenv/lib/python2.7/site-packages/mailpile\n\tcd mp-virtualenv/lib/python2.7/site-packages/ && ln -s ../../../../mailpile\n\trm -rf mp-virtualenv/share/mailpile\n\tcd mp-virtualenv/share/ && ln -s ../../shared-data mailpile\n\t@touch mp-virtualenv/bin/.dev\n\nbower_components:\n\t@bower install\n\njs: bower_components\n\t# Warning: Horrible hack to extract rules from Gruntfile.js\n\t@rm -f shared-data/default-theme/js/libraries.min.js\n\t@rm -f shared-data/default-theme/js/mailpile-min.js.tmp*\n\t@cat Gruntfile.js \\\n |sed -e '1,/concat:/d ' \\\n |sed -e '1,/src:/d' -e '/dest:/,$$d' \\\n |grep / \\\n |sed -e \"s/[',]/ /g\" \\\n |xargs sed -e '$$a;' \\\n >> shared-data/default-theme/js/mailpile-min.js.tmp\n\t@uglify -s shared-data/default-theme/js/mailpile-min.js.tmp \\\n -o shared-data/default-theme/js/mailpile-min.js.tmp2\n\t@sed -e \"s/@MP_JSBUILD_INFO@/`./scripts/gitwhere.sh`/\" \\\n\t < shared-data/default-theme/js/libraries.js \\\n\t > shared-data/default-theme/js/libraries.min.js\n\t@echo '/* Sources...' \\\n\t >> shared-data/default-theme/js/libraries.min.js\n\t@bower --offline --no-color list \\\n\t >> shared-data/default-theme/js/libraries.min.js\n\t@echo '*/' \\\n\t >> shared-data/default-theme/js/libraries.min.js\n\t@cat shared-data/default-theme/js/mailpile-min.js.tmp2 \\\n >> shared-data/default-theme/js/libraries.min.js\n\t@rm -f shared-data/default-theme/js/mailpile-min.js.tmp*\n\nless: less-compiler bower_components\n\t@cp -fa \\\n bower_components/select2/select2.png \\\n bower_components/select2/select2x2.png \\\n bower_components/select2/select2-spinner.gif \\\n shared-data/default-theme/css/\n\t@make -s -f scripts/less-compiler.mk\n\nless-loop: less-compiler\n\t@echo 'Running less compiler every 15 seconds. CTRL+C quits.'\n\t@while [ 1 ]; do \\\n make -s less; \\\n sleep 15; \\\n done\n\nless-compiler:\n\tbower install\n\t@cp scripts/less-compiler.in scripts/less-compiler.mk\n\t@find shared-data/default-theme/less/ -name '*.less' \\\n |perl -npe s'/^/\\t/' \\\n\t\t|perl -npe 's/$$/\\\\/' \\\n >>scripts/less-compiler.mk\n\t@echo >> scripts/less-compiler.mk\n\t@perl -e 'print \"\\t\\@touch .less-deps\", $/' >> scripts/less-compiler.mk\n\ngenmessages:\n\t@scripts/make-messages.sh\n\ncompilemessages:\n\t@scripts/compile-messages.sh\n\ntransifex:\n\ttx pull -a --minimum-perc=25\n\ttx pull -l is,en_GB\n\n\n# -----------------------------------------------------------------------------\n# BUILD\n# -----------------------------------------------------------------------------\n\ndist/version.txt: mailpile/config/defaults.py scripts/version.py\n\tmkdir -p dist\n\tscripts/version.py > dist/version.txt\n\ndist/mailpile.tar.gz: mrproper genmessages transifex dist/version.txt\n\tgit submodule update --init --recursive\n\tgit submodule foreach 'git reset --hard && git clean -dfx'\n\tmkdir -p dist\n\tscripts/version.py > dist/version.txt\n\ttar --exclude='./packages/debian' --exclude=dist --exclude-vcs -czf dist/mailpile-$$(cat dist/version.txt).tar.gz -C $(shell pwd) .\n\t(cd dist; ln -fs mailpile-$$(cat version.txt).tar.gz mailpile.tar.gz)\n\n.dockerignore: dist/version.txt packages/Dockerfile_debian packages/debian packages/debian/rules\n\tmkdir -p dist\n\tdocker build \\\n\t --file=packages/Dockerfile_debian \\\n\t --tag=mailpile-deb-builder \\\n\t ./\n\ttouch .dockerignore\n\ndpkg: dist/mailpile.tar.gz .dockerignore\n\tdocker run \\\n\t --rm --volume=$$(pwd)/dist:/mnt/dist \\\n\t mailpile-deb-builder\n" }, { "path": "README.md", "content": "# Welcome to Mailpile! #\n\n**IMPORTANT NOTE**\n\nDevelopment on this codebase has halted, until the\n[Python3 rewrite](https://community.mailpile.is/t/a-very-uninformative-progress-update-mailpile-2/785)\nhas completed.\n\nApologies to those who have unanswered, out-standing pull requests and\nissues. 😢 Your efforts are appreciated!\n\nIf you rely on this code and have your own branch which you actively\nmaintain, let us know: we would be happy to link to it.\n\nIf you need to run Mailpile v1 to access legacy data, consider using\nour [legacy Docker images](https://github.com/mailpile/Mailpile-v1-Docker).\n\n\n------------------------------------------------------------------------\n\n## Introduction (Obsolete) ##\n\nMailpile () is a modern, fast web-mail client\nwith user-friendly encryption and privacy features. The development of\nMailpile is funded by\n[a large community of backers](https://www.mailpile.is/#community)\nand all code related to the project is and will be released under an OSI\napproved Free Software license.\n\nMailpile places great emphasis on providing a clean, elegant user\ninterface and pleasant user experience. In particular, Mailpile aims to\nmake it easy and convenient to receive and send PGP encrypted or signed\ne-mail.\n\nMailpile's primary user interface is web-based, but it also has a basic\ncommand-line interface and an API for developers. Using web technology\nfor the interface allows Mailpile to function both as a local desktop\napplication (accessed by visiting `localhost` in the browser) or a\nremote web-mail on a personal server or VPS.\n\nThe core of Mailpile is a fast search engine, custom written to deal\nwith large volumes of e-mail on consumer hardware. The search engine\nallows e-mail to be organized using tags (similar to GMail's labels) and\nthe application can be configured to automatically tag incoming mail\neither based on static rules or bayesian classifiers.\n\n\n### Trying Mailpile\n\nIf you need to run Mailpile v1 to access legacy data, consider using\nour [legacy Docker images](https://github.com/mailpile/Mailpile-v1-Docker).\n\n\n## Credits and License ##\n\nBjarni R. Einarsson () created this! If you\nthink it's neat, you should also check out PageKite:\n. [Smári]() and\n[Brennan](https://brennannovak.com) joined the team in 2013 and made\nthis a real project (not just a toy search engine).\n\nThe original GMail team deserve a mention for their inspiring work:\nwishing the Free Software world had something like GMail is what\nmotivated Bjarni to start working on Mailpile. We would also like to\nthank Edward Snowden for inspiring us to try and make PGP usable for\njournalists and everday folks!\n\nContributors:\n\n- Bjarni R. Einarsson ()\n- Brennan Novak ()\n- Smari McCarthy ()\n- Lots more, run `git shortlog -s` for a list! (Or check\n [GitHub](https://github.com/mailpile/Mailpile/graphs/contributors).)\n\nAnd of course, we couldn't do this without [our community of\nbackers](https://www.mailpile.is/#community).\n\nThis program is free software: you can redistribute it and/or modify it\nunder the terms of the GNU Affero General Public License as published by\nthe Free Software Foundation. See the file `COPYING.md` for details.\n\n" }, { "path": "babel.cfg", "content": "# Extraction from Python source files\n[python: mailpile/**.py]\nencoding = utf-8\n\n[python: shared-data/contrib/**.py]\nencoding = utf-8\n\n[python: shared-data/multiple/**.py]\nencoding = utf-8\n\n[python: shared-data/mailpile-gui/**.py]\nencoding = utf-8\n\n# Extraction from Jinja2 template files\n[jinja2: shared-data/default-theme/html/**]\nencoding = utf-8\nignore_tags = style\nextensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand\nsilent = false\n\n[jinja2: shared-data/contrib/html/**.html]\nencoding = utf-8\nignore_tags = style\nextensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand\nsilent = false\n\n[jinja2: shared-data/contrib/html/**.js]\nencoding = utf-8\nignore_tags = style\nextensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand\nsilent = false\n" }, { "path": "bower.json", "content": "{\n \"name\": \"mailpile\",\n \"version\": \"0.1.0\",\n \"homepage\": \"https://mailpile.is\",\n \"authors\": [\n \"Various\"\n ],\n \"description\": \"Front end libraries and dependencies for Mailpile default theme\",\n \"main\": \"libraries.js\",\n \"license\": \"Various\",\n \"private\": true,\n \"ignore\": [\n \"**/.*\",\n \"node_modules\",\n \"bower_components\",\n \"test\",\n \"tests\"\n ],\n \"dependencies\": {\n \"underscore\": \">=1.7.0\",\n \"bootstrap\": \"~3.3.0\",\n \"favico.js\": \"~0.3.5\",\n \"html5shiv\": \"~3.7.0\",\n \"jquery\": \"~2.1\",\n \"autosize\": \"~3.0.14\",\n \"jquery-confirm\": \"~2.1.1\",\n \"jquery-slugify\": \"~1.0.3\",\n \"jquery-timer\": \"*\",\n \"jquery.ui\": \"~1.10\",\n \"less-elements-old\": \"~1.0\",\n \"mousetrap\": \"~1.6.0\",\n \"moxie\": \"~1.3.4\",\n \"plupload\": \"~2.1.2\",\n \"qtip2\": \"~2.1.1\",\n \"rebar\": \"~0.3.2\",\n \"select2\": \"3.5.4\",\n \"typeahead.js\": \"~0.10.5\",\n \"jqueryui-touch-punch\": \"*\",\n \"dompurify\": \"~1.0.11\"\n }\n}\n" }, { "path": "docker-compose.dev.yml", "content": "version: '3.0'\nservices:\n mailpile_dev:\n tty: true\n stdin_open: true\n container_name: mailpile_dev\n build:\n context: .\n dockerfile: Dockerfile.dev\n image: mailpile_dev\n volumes:\n - .:/Mailpile\n - .dev-mailpile-data:/mailpile-data:rw\n ports:\n - 33412:33411\n\n" }, { "path": "docker-compose.yml", "content": "version: '3.0'\nservices:\n mailpile:\n container_name: mailpile\n build: .\n image: mailpile\n volumes:\n - .:/Mailpile\n - .dev-mailpile-data:/mailpile-data\n ports:\n - 33411:33411\n\n" }, { "path": "install_hooks.py", "content": "import os\nimport sys\n\ndef symlink_develop(config):\n if 'develop' in sys.argv:\n share_path = os.path.join(sys.prefix, 'share')\n\n if not os.path.exists(share_path):\n os.makedirs(share_path)\n\n os.symlink(\n os.path.join(\n os.path.dirname(os.path.realpath(__file__)),\n 'shared-data'\n ),\n os.path.join(sys.prefix, 'share', 'mailpile')\n )\n" }, { "path": "mailpile/__init__.py", "content": "from mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\n__all__ = ['Mailpile',\n \"app\", \"commands\", \"plugins\", \"mailutils\", \"search\", \"ui\", \"util\"]\n\n\nclass Mailpile(object):\n \"\"\"This object provides a simple Python API to Mailpile.\"\"\"\n\n def __init__(self,\n ui=None,\n workdir=None,\n session=None):\n import mailpile.app\n import mailpile.config.defaults\n import mailpile.ui\n if not session:\n ui = ui or mailpile.ui.UserInteraction\n self._config = mailpile.app.ConfigManager(\n workdir=workdir, rules=mailpile.config.defaults.CONFIG_RULES)\n self._session = mailpile.ui.Session(self._config)\n self._ui = self._session.ui = ui(self._config)\n self._session.config.load(self._session)\n self._session.main = True\n else:\n self._session = session\n self._config = session.config\n self._ui = session.ui\n\n for cls in mailpile.commands.COMMANDS:\n names, argspec = cls.SYNOPSIS[1:3], cls.SYNOPSIS[3]\n if names[0]:\n setattr(self, *self._mk_action(cls, names[0], argspec))\n if names[1] and (names[0] != names[1]):\n setattr(self, *self._mk_action(cls, names[1], argspec))\n\n def _mk_action(self, cls, cmd, argspec):\n import mailpile.commands\n if argspec:\n\n def fnc(*args, **kwargs):\n return mailpile.commands.Action(self._session, cmd, args,\n data=kwargs)\n else:\n\n def fnc(**kwargs):\n return mailpile.commands.Action(self._session, cmd, '',\n data=kwargs)\n\n fnc.__doc__ = '%s(%s) # %s' % (cmd, argspec or '', cls.__doc__)\n return cmd.replace('/', '_'), fnc\n\n def Interact(self):\n import mailpile.util\n mailpile.util.QUITTING = False\n self._session.interactive = self._session.ui.interactive = True\n try:\n self._session.config.prepare_workers(self._session, daemons=True)\n mailpile.app.Interact(self._session)\n except KeyboardInterrupt:\n pass\n finally:\n mailpile.util.QUITTING = mailpile.util.QUITTING or True\n self._session.config.stop_workers()\n self._session.interactive = self._session.ui.interactive = False\n" }, { "path": "mailpile/__main__.py", "content": "import sys\nfrom mailpile.app import Main\n\n\ndef main():\n Main(sys.argv[1:])\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "mailpile/app.py", "content": "from __future__ import print_function\nimport getopt\nimport gettext\nimport locale\nimport os\nimport re\nimport sys\nimport traceback\n\nimport mailpile.util\nimport mailpile.config.defaults\nimport mailpile.platforms\nfrom mailpile.commands import COMMANDS, Command, Action\nfrom mailpile.config.manager import ConfigManager\nfrom mailpile.conn_brokers import DisableUnbrokeredConnections\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.plugins import PluginManager\nfrom mailpile.plugins.core import Help, HelpSplash, HealthCheck\nfrom mailpile.plugins.core import Load, Rescan, Quit\nfrom mailpile.plugins.motd import MessageOfTheDay\nfrom mailpile.plugins.setup_magic import Setup\nfrom mailpile.ui import ANSIColors, Session, UserInteraction, Completer\nfrom mailpile.util import *\n\n_plugins = PluginManager(builtin=__file__)\n\n# This makes sure mailbox \"plugins\" get loaded... has to go somewhere?\nfrom mailpile.mailboxes import *\n\n# This is also a bit silly, should be somewhere else?\nHelp.ABOUT = mailpile.config.defaults.ABOUT\n\n# We may try to load readline later on... maybe?\nreadline = None\n\n\n##[ Main ]####################################################################\n\n\ndef threaded_raw_input(prompt):\n \"\"\"These shenigans are necessary to let Quit work reliably.\"\"\"\n def reader(container):\n try:\n line = raw_input(prompt).decode('utf-8').strip()\n container.append(line)\n except EOFError:\n pass\n o = []\n t = threading.Thread(target=reader, args=(o,))\n t.daemon = True\n t.start()\n while t.isAlive() and not mailpile.util.QUITTING:\n t.join(timeout=1)\n if not o:\n raise EOFError()\n return o[0]\n\n\ndef write_readline_history(session):\n try:\n if session.config.sys.history_length > 0:\n readline.write_history_file(session.config.history_file())\n else:\n safe_remove(session.config.history_file())\n except (OSError, AttributeError, IOError):\n pass\n\n\ndef CatchUnixSignals(session):\n def quit_app(sig, stack):\n Quit(session, 'quit').run()\n\n def reload_app(sig, stack):\n pass\n\n try:\n import signal\n if os.name != 'nt':\n signal.signal(signal.SIGTERM, quit_app)\n signal.signal(signal.SIGQUIT, quit_app)\n signal.signal(signal.SIGUSR1, reload_app)\n else:\n signal.signal(signal.SIGTERM, quit_app)\n except ImportError:\n pass\n\n\ndef FriendlyPipeTransform(session, opt):\n \"\"\"Shell-like syntax to invoke the pipe command or change render modes.\"\"\"\n old_render_mode = None\n if session.config.prefs.friendly_pipes:\n if ' |' in opt:\n cmd, pipe = opt.split(' |', 1)\n opt = 'pipe %s -- %s' % (pipe.strip(), cmd.strip())\n elif re.search(' >\\S+$', opt):\n cmd, pipe = opt.rsplit(' >', 1)\n opt = 'pipe >%s -- %s' % (pipe, cmd.strip())\n opt = opt.replace('\\\\|', '|').replace('\\\\>', '>')\n\n if re.search(' :(json|j?html|text|\\S+.html(?:!\\S+))$', opt):\n old_render_mode = session.ui.render_mode\n opt, session.ui.render_mode = opt.rsplit(' :', 1)\n\n return old_render_mode, opt\n\n\ndef Interact(session):\n global readline\n try:\n import readline as rl # Unix-only\n readline = rl\n except ImportError:\n pass\n\n try:\n if readline:\n readline.read_history_file(session.config.history_file())\n readline.set_completer_delims(Completer.DELIMS)\n readline.set_completer(Completer(session).get_completer())\n for opt in [\"tab: complete\", \"set show-all-if-ambiguous on\"]:\n readline.parse_and_bind(opt)\n except IOError:\n pass\n\n # Negative history means no saving state to disk.\n history_length = session.config.sys.history_length\n if readline is None:\n pass # history currently not supported under Windows / Mac\n elif history_length >= 0:\n readline.set_history_length(history_length)\n else:\n readline.set_history_length(-history_length)\n\n try:\n prompt = session.ui.term.color('mailpile> ',\n color=session.ui.term.BLACK,\n weight=session.ui.term.BOLD,\n readline=(readline is not None))\n while not mailpile.util.QUITTING:\n try:\n with session.ui.term:\n if Setup.Next(session.config, 'anything') != 'anything':\n session.ui.notify(\n _('Mailpile is unconfigured, please run `setup`'\n ' or visit the web UI.'))\n session.ui.block()\n opt = threaded_raw_input(prompt)\n except KeyboardInterrupt:\n session.ui.unblock(force=True)\n session.ui.notify(_('Interrupted. '\n 'Press CTRL-D or type `quit` to quit.'))\n continue\n session.ui.term.check_max_width()\n session.ui.unblock(force=True)\n if opt:\n old_render_mode, opt = FriendlyPipeTransform(session, opt)\n if ' ' in opt:\n opt, arg = opt.split(' ', 1)\n else:\n arg = ''\n try:\n result = Action(session, opt, arg)\n session.ui.block()\n session.ui.display_result(result)\n except UsageError as e:\n session.fatal_error(unicode(e))\n except UrlRedirectException as e:\n session.fatal_error('Tried to redirect to: %s' % e.url)\n if old_render_mode is not None:\n session.ui.render_mode = old_render_mode\n except EOFError:\n print()\n finally:\n session.ui.unblock(force=True)\n\n write_readline_history(session)\n\n\nclass InteractCommand(Command):\n SYNOPSIS = (None, 'interact', None, None)\n ORDER = ('Internals', 2)\n CONFIG_REQUIRED = False\n RAISES = (KeyboardInterrupt,)\n\n def command(self):\n session, config = self.session, self.session.config\n\n session.interactive = True\n if mailpile.platforms.TerminalSupportsAnsiColors():\n session.ui.term = ANSIColors()\n\n # Ensure we have a working GnuPG\n self._gnupg().common_args(will_send_passphrase=True)\n\n # Create and start the rest of the threads, load the index.\n if config.loaded_config:\n Load(session, '').run(quiet=True)\n else:\n config.prepare_workers(session, daemons=True)\n\n # Note: We do *not* update the MOTD on startup, to keep things\n # fast, and to avoid leaking our IP on setup, before Tor\n # has been configured.\n splash = HelpSplash(session, 'help', []).run()\n motd = MessageOfTheDay(session, 'motd', ['--noupdate']).run()\n session.ui.display_result(splash)\n print() # FIXME: This is a hack!\n session.ui.display_result(motd)\n\n Interact(session)\n\n return self._success(_('Ran interactive shell'))\n\n\nclass WaitCommand(Command):\n SYNOPSIS = (None, 'wait', None, None)\n ORDER = ('Internals', 2)\n CONFIG_REQUIRED = False\n RAISES = (KeyboardInterrupt,)\n\n def command(self):\n self.session.ui.display_result(HelpSplash(self.session, 'help', []\n ).run(interactive=False))\n while not mailpile.util.QUITTING:\n time.sleep(1)\n return self._success(_('Did nothing much for a while'))\n\n\ndef Main(args):\n try:\n mailpile.platforms.DetectBinaries(_raise=OSError)\n except OSError as e:\n binary = str(e).split()[0]\n sys.stderr.write(\"\"\"\nRequired binary missing or unusable: %s\n\nIf you know where it is, or would like to skip this test and run Mailpile\nanyway, you can set one of the following environment variables:\n\n MAILPILE_%s=\"/path/to/binary\"\nor\n MAILPILE_IGNORE_BINARIES=\"%s\" # Can be a space-separated list\n\nNote that skipping a binary check may cause the app to become unstable or\nfail in unexpected ways. If it breaks you get to keep both pieces!\n\n\"\"\" % (e, binary.upper(), binary))\n sys.exit(1)\n\n # Enable our connection broker, try to prevent badly behaved plugins from\n # bypassing it.\n DisableUnbrokeredConnections()\n\n # Bootstrap translations until we've loaded everything else\n mailpile.i18n.ActivateTranslation(None, ConfigManager, None)\n try:\n # Create our global config manager and the default (CLI) session\n config = ConfigManager(rules=mailpile.config.defaults.CONFIG_RULES)\n session = Session(config)\n cli_ui = session.ui = UserInteraction(config)\n session.main = True\n try:\n CatchUnixSignals(session)\n config.clean_tempfile_dir()\n config.load(session)\n except IOError:\n if config.sys.debug:\n session.ui.error(_('Failed to decrypt configuration, '\n 'please log in!'))\n HealthCheck(session, None, []).run()\n config.prepare_workers(session)\n except AccessError as e:\n session.ui.error('Access denied: %s\\n' % e)\n sys.exit(1)\n\n try:\n try:\n if '--login' in args:\n a1 = args[:args.index('--login') + 1]\n a2 = args[len(a1):]\n else:\n a1, a2 = args, []\n\n allopts = []\n for argset in (a1, a2):\n shorta, longa = '', []\n for cls in COMMANDS:\n shortn, longn, urlpath, arglist = cls.SYNOPSIS[:4]\n if arglist:\n if shortn:\n shortn += ':'\n if longn:\n longn += '='\n if shortn:\n shorta += shortn\n if longn:\n longa.append(longn.replace(' ', '_'))\n\n opts, args = getopt.getopt(argset, shorta, longa)\n allopts.extend(opts)\n for opt, arg in opts:\n session.ui.display_result(Action(\n session, opt.replace('-', ''), arg.decode('utf-8')))\n if args:\n session.ui.display_result(Action(\n session, args[0], ' '.join(args[1:]).decode('utf-8')))\n\n except (getopt.GetoptError, UsageError) as e:\n session.fatal_error(unicode(e))\n\n if (not allopts) and (not a1) and (not a2):\n InteractCommand(session).run()\n\n except KeyboardInterrupt:\n pass\n\n except:\n traceback.print_exc()\n\n finally:\n write_readline_history(session)\n\n # Make everything in the background quit ASAP...\n mailpile.util.LAST_USER_ACTIVITY = 0\n mailpile.util.QUITTING = mailpile.util.QUITTING or True\n\n if config.plugins:\n config.plugins.process_shutdown_hooks()\n\n config.stop_workers()\n if config.index:\n config.index.save_changes()\n if config.event_log:\n config.event_log.close()\n\n session.ui.display_result(Action(session, 'cleanup', ''))\n\n if session.interactive and config.sys.debug:\n session.ui.display_result(Action(session, 'ps', ''))\n\n # Remove anything that we couldn't remove before\n safe_remove()\n\n # Restart the app if that's what was requested\n if mailpile.util.QUITTING == 'restart':\n os.execv(sys.argv[0], sys.argv)\n\n\n_plugins.register_commands(InteractCommand, WaitCommand)\n\nif __name__ == \"__main__\":\n Main(sys.argv[1:])\n" }, { "path": "mailpile/auth.py", "content": "import time\nfrom urlparse import parse_qs, urlparse\nfrom urllib import quote, urlencode\n\nfrom mailpile.commands import Command\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.plugins import PluginManager\nfrom mailpile.security import SecurePassphraseStorage\nfrom mailpile.util import *\n\nGLOBAL_LOGIN_LOCK = CryptoLock()\n\n\nclass UserSession(object):\n EXPIRE_AFTER = 7 * 24 * 3600\n\n def __init__(self, ts=None, auth=None, data=None):\n self.ts = ts or time.time()\n self.auth = auth\n self.data = data or {}\n\n def is_expired(self, now=None):\n return (self.ts < (now or time.time()) - self.EXPIRE_AFTER)\n\n def update_ts(self):\n self.ts = time.time()\n\n\nclass UserSessionCache(dict):\n def delete_expired(self, now=None):\n now = now or time.time()\n for k in self.keys():\n if self[k].is_expired(now=now):\n del self[k]\n\n\ndef VerifyAndStorePassphrase(config, passphrase=None, sps=None,\n key=None):\n if passphrase and not sps:\n sps = SecurePassphraseStorage(passphrase)\n passphrase = 'this probably does not really overwrite :-( '\n\n safe_assert(\n (sps is not None) and\n (config.load_master_key(sps)))\n\n # Fun side effect: changing the passphrase invalidates the message cache\n import mailpile.mailutils.emails\n mailpile.mailutils.emails.ClearParseCache(full=True)\n\n return sps\n\n\ndef SetLoggedIn(cmd, user=None, redirect=False, session_id=None):\n user = user or 'DEFAULT'\n\n sid = session_id or cmd.session.ui.html_variables.get('http_session')\n if sid:\n if cmd:\n cmd.session.ui.debug('Logged in %s as %s' % (sid, user))\n SESSION_CACHE[sid] = UserSession(auth=user, data={\n 't': '%x' % int(time.time()),\n })\n\n if cmd:\n if redirect:\n return cmd._do_redirect()\n else:\n return True\n\n\ndef CheckPassword(config, username, password):\n # FIXME: Do something with the username\n username = username or 'DEFAULT'\n sps = config.passphrases and config.passphrases.get(username)\n return sps.compare(password) and username\n\n\ndef IndirectPassword(config, pwd):\n pp = pwd.split(':')\n if len(pp) > 2 and pp[0] == '_SECRET_':\n if pp[1] in config.secrets:\n return config.secrets[pp[1]].password\n if pp[1] in config.passphrases:\n return config.passphrases[pp[1]].get_passphrase()\n return pwd\n\n\nSESSION_CACHE = UserSessionCache()\nLOGIN_FAILURES = []\n\n\ndef LogoutAll():\n for k in list(SESSION_CACHE.keys()):\n del SESSION_CACHE[k]\n\n\nclass Authenticate(Command):\n \"\"\"Authenticate a user (log in)\"\"\"\n SYNOPSIS = (None, 'login', 'auth/login', None)\n ORDER = ('Internals', 5)\n SPLIT_ARG = False\n IS_INTERACTIVE = True\n\n CONFIG_REQUIRED = False\n HTTP_AUTH_REQUIRED = False\n HTTP_STRICT_VARS = False\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_POST_VARS = {\n 'user': 'User to authenticate as',\n 'pass': 'Password or passphrase'\n }\n\n @classmethod\n def RedirectBack(cls, url, data):\n qs = [(k, v.encode('utf-8')) for k, vl in data.iteritems() for v in vl\n if k not in ['_method', '_path'] + cls.HTTP_POST_VARS.keys()]\n qs = urlencode(qs)\n url = ''.join([url, '?%s' % qs if qs else ''])\n raise UrlRedirectException(url)\n\n def _result(self, result=None):\n global LOGIN_FAILURES\n result = result or {}\n result['login_banner'] = self.session.config.sys.login_banner\n result['login_failures'] = LOGIN_FAILURES\n return result\n\n def _error(self, message, info=None, result=None):\n global LOGIN_FAILURES\n LOGIN_FAILURES.append(int(time.time()))\n time.sleep(min(5, 0.1 + len(LOGIN_FAILURES) / 2))\n return Command._error(self, message,\n info=info, result=self._result(result))\n\n def _success(self, message, result=None):\n return Command._success(self, message, result=self._result(result))\n\n def _do_redirect(self):\n path = self.data.get('_path', [None])[0]\n\n # These are here to prevent people from abusing this to redirect to\n # arbitrary URLs on the Internet.\n if path:\n url = urlparse(path)\n safe_assert(not url.scheme and not url.netloc)\n\n if (path and\n not path[1:].startswith(DeAuthenticate.SYNOPSIS[2] or '!') and\n not path[1:].startswith(self.SYNOPSIS[2] or '!')):\n self.RedirectBack(self.session.config.sys.http_path + path, self.data)\n else:\n raise UrlRedirectException('%s/' % self.session.config.sys.http_path)\n\n def _do_login(self, user, password, load_index=False, redirect=False):\n global LOGIN_FAILURES\n session, config = self.session, self.session.config\n session_id = self.session.ui.html_variables.get('http_session')\n\n # This prevents folks from sending us a DEFAULT user (upper case),\n # which is an internal security bypass below.\n user = user and user.lower()\n\n if not user:\n try:\n # Verify the passphrase\n if CheckPassword(config, None, password):\n sps = config.passphrases['DEFAULT']\n else:\n sps = VerifyAndStorePassphrase(config, passphrase=password)\n\n if sps:\n # Load the config and index, if necessary\n config = self._config()\n self._idx(wait=False)\n if load_index:\n try:\n while not config.index:\n time.sleep(1)\n except KeyboardInterrupt:\n pass\n\n session.ui.debug('Good passphrase for %s' % session_id)\n self.record_user_activity()\n LOGIN_FAILURES = []\n return self._success(_('Hello world, welcome!'), result={\n 'authenticated': SetLoggedIn(self, redirect=redirect)\n })\n else:\n session.ui.debug('No GnuPG, checking DEFAULT user')\n # No GnuPG, see if there is a DEFAULT user in the config\n user = 'DEFAULT'\n\n except (AssertionError, IOError):\n session.ui.debug('Bad passphrase for %s' % session_id)\n return self._error(_('Invalid password, please try again'))\n\n if user in config.logins or user == 'DEFAULT':\n # FIXME: Salt and hash the password, check if it matches\n # the entry in our user/password list (TODO).\n # NOTE: This hack effectively disables auth without GnUPG\n if user == 'DEFAULT':\n session.ui.debug('FIXME: Unauthorized login allowed')\n return self._logged_in(redirect=redirect)\n raise Exception('FIXME')\n\n return self._error(_('Incorrect username or password'))\n\n def command(self):\n session_id = self.session.ui.html_variables.get('http_session')\n\n if self.data.get('_method', '') == 'POST':\n if 'pass' in self.data:\n with GLOBAL_LOGIN_LOCK:\n return self._do_login(self.data.get('user', [None])[0],\n self.data['pass'][0],\n redirect=True)\n\n elif not self.data:\n password = self.session.ui.get_password(_('Your password: '))\n return self._do_login(None, password, load_index=True)\n\n elif (session_id in SESSION_CACHE and\n SESSION_CACHE[session_id].auth and\n '_method' in self.data):\n self._do_redirect()\n\n return self._success(_('Please log in'))\n\n\nclass DeAuthenticate(Command):\n \"\"\"De-authenticate a user (log out)\"\"\"\n SYNOPSIS = (None, 'logout', 'auth/logout', '[]')\n ORDER = ('Internals', 5)\n SPLIT_ARG = False\n IS_INTERACTIVE = True\n CONFIG_REQUIRED = False\n HTTP_AUTH_REQUIRED = False\n HTTP_CALLABLE = ('GET', 'POST')\n\n def command(self):\n # FIXME: Should this only be a POST request?\n # FIXME: This needs CSRF protection.\n\n session_id = self.session.ui.html_variables.get('http_session')\n if self.args and not session_id:\n session_id = self.args[0]\n\n if session_id:\n try:\n self.session.ui.debug('Logging out %s' % session_id)\n del SESSION_CACHE[session_id]\n return self._success(_('Goodbye!'))\n except KeyError:\n pass\n\n return self._error(_('No session found!'))\n\n\nclass SetPassphrase(Command):\n \"\"\"Manage storage of passwords (passphrases)\"\"\"\n SYNOPSIS = (None, 'set/password', 'settings/set/password',\n ' [store|cache-only[:]|fail|forget]')\n ORDER = ('Config', 9)\n SPLIT_ARG = True\n IS_INTERACTIVE = True\n IS_USER_ACTIVITY = True\n CONFIG_REQUIRED = True\n HTTP_AUTH_REQUIRED = True\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = {\n 'id': 'KeyID or account name',\n 'mailsource': 'Mail source ID',\n 'mailroute': 'Mail route ID',\n 'is_locked': 'Assume key is locked'}\n HTTP_POST_VARS = {\n 'password': 'KeyID or account name',\n 'policy-ttl': 'Combined policy and TTL',\n 'policy': 'store|cache-only|fail|forget',\n 'ttl': 'Seconds after which it expires, -1 = never',\n 'update_mailsources': 'If true, update mail source settings',\n 'update_mailroutes': 'If true, update mail route settings',\n 'redirect': 'URL to redirect to on success'}\n\n def _get_profiles(self):\n return self.session.config.vcards.find_vcards([], kinds=['profile'])\n\n def _get_policy(self, fingerprint):\n if fingerprint in self.session.config.secrets:\n return self.session.config.secrets[fingerprint].policy\n elif fingerprint in self.session.config.passphrases:\n return 'cache-only'\n return None\n\n def _massage_key_info(self, fingerprint, key_info, profiles=None, is_locked=False):\n config = self.session.config\n fingerprint = fingerprint.lower()\n\n key_info[\"uids\"].sort(\n key=lambda k: (k.get(\"name\"), k.get(\"email\"), k.get(\"comment\")))\n\n if not is_locked:\n key_info['policy'] = self._get_policy(fingerprint)\n if key_info['policy'] is None:\n del key_info['policy']\n\n key_info[\"accounts\"] = []\n if profiles is None:\n profiles = self._get_profiles()\n for vc in profiles:\n vc_pgp_key = (vc.pgp_key or '').lower()\n if vc_pgp_key == fingerprint:\n key_info[\"accounts\"].append({\n 'name': vc.fn,\n 'email': vc.email,\n 'rid': vc.random_uid})\n\n return key_info\n\n def _lookup_key(self, keyid, **kwargs):\n keylist = self._gnupg().list_secret_keys(selectors=[keyid])\n if len(keylist) != 1:\n raise ValueError(\"Too many or too few keys found!\")\n fingerprint, key_info = keylist.keys()[0], keylist.values()[0]\n return fingerprint, self._massage_key_info(\n fingerprint, key_info, **kwargs)\n\n def _list_keys(self, **kwargs):\n keylist = self._gnupg().list_secret_keys()\n profiles = self._get_profiles()\n for fingerprint, key_info in keylist.iteritems():\n self._massage_key_info(fingerprint, key_info,\n profiles=profiles, **kwargs)\n return keylist\n\n def _get_account(self, cfg):\n username = cfg.username\n if cfg.username and '@' not in cfg.username:\n username = '%s@%s' % (username, cfg.host)\n return username\n\n def _user_fingerprint(self, username):\n return username.replace('@', '_').replace('.', '_').lower()\n\n def _list_accounts(self, only=None):\n accounts = {}\n def _add_account(cfg, which):\n username = self._get_account(cfg)\n if (username\n and ((username == only) or only is None)\n and cfg.auth_type == 'password'):\n if username in accounts:\n accounts[username][which] = cfg.host\n else:\n fingerprint = self._user_fingerprint(username)\n accounts[username] = {\n which: cfg.host,\n 'username': username,\n 'policy': self._get_policy(fingerprint)}\n if accounts[username]['policy'] is None:\n del accounts[username]['policy']\n\n for msid, route in self.session.config.routes.iteritems():\n _add_account(route, 'route')\n for msid, source in self.session.config.sources.iteritems():\n _add_account(source, 'source')\n\n return accounts\n\n def _account_details(self, account):\n return self._list_accounts(only=account).get(account)\n\n def _check_master_password(self, password, account=None, fingerprint=None):\n return CheckPassword(self.session.config, None, password)\n\n def _check_password(self, password, account=None, fingerprint=None):\n if account:\n # We're going to keep punting on this for a while...\n return True\n elif fingerprint:\n sps = SecurePassphraseStorage(password)\n gpg = GnuPG(self.session.config)\n status, sig = gpg.sign('OK', fromkey=fingerprint, passphrase=sps)\n return (status == 0)\n else:\n return True\n\n def _prepare_result(self, account=None, keyid=None, is_locked=False):\n if account:\n fingerprint = self._user_fingerprint(account)\n result = {'account': self._account_details(account)}\n elif keyid:\n fingerprint, info = self._lookup_key(keyid, is_locked=is_locked)\n result = {'key': info}\n else:\n fingerprint = None\n result = {\n 'keylist': self._list_keys(is_locked=is_locked),\n 'accounts': self._list_accounts()}\n return fingerprint, result\n\n def command(self):\n config = self.session.config\n\n policyttl = self.args[1] if (len(self.args) > 1) else 'cache-only:-1'\n is_locked = self.data.get('is_locked', [0])[0]\n if 'policy-ttl' in self.data:\n policyttl = self.data['policy-ttl'][0]\n if ':' in policyttl:\n policy, ttl = policyttl.split(':')\n else:\n policy, ttl = policyttl, -1\n if 'policy' in self.data:\n policy = self.data['policy'][0]\n if 'ttl' in self.data:\n ttl = self.data['policy'][0]\n ttl = float(ttl)\n\n fingerprint = info = account = keyid = None\n which = self.args[0] if self.args else self.data.get('id', [None])[0]\n if which and '@' in which:\n account = which\n else:\n keyid = which\n\n if not account and not keyid:\n msid = self.data.get('mailsource', [None])[0]\n if msid:\n account = self._get_account(config.sources[msid])\n mrid = self.data.get('mailroute', [None])[0]\n if mrid:\n account = self._get_account(config.routes[mrid])\n\n fingerprint, result = self._prepare_result(\n account=account, keyid=keyid, is_locked=is_locked)\n\n if policy in ('display', 'unprotect'):\n pass_prompt = _('Enter your Mailpile password')\n pass_check = self._check_master_password\n else:\n pass_prompt = _('Enter your password')\n pass_check = self._check_password\n\n if self.data.get('_method', None) == 'GET':\n return self._success(pass_prompt, result)\n\n safe_assert(fingerprint is not None)\n fingerprint = fingerprint.lower()\n if fingerprint in config.secrets:\n if config.secrets[fingerprint].policy == 'protect':\n if policy not in ('unprotect', 'display'):\n result['error'] = _('That key is managed by Mailpile,'\n ' it cannot be changed directly.')\n return self._error(_('Protected secret'), result=result)\n\n if self.data.get('_method', None) == 'POST':\n password = self.data.get('password', [None])[0]\n update_ms = self.data.get('update_mailsources', [False])[0]\n update_mr = self.data.get('update_mailroutes', [False])[0]\n else:\n password = self.session.ui.get_password(pass_prompt + ': ')\n update_ms = update_mr = (account is not None)\n\n if update_ms or update_mr:\n safe_assert(account is not None)\n\n if not pass_check(password, account=account, fingerprint=fingerprint):\n result['error'] = _('Password incorrect! Try again?')\n return self._error(_('Incorrect password'), result=result)\n\n def _account_matches(cfg):\n return (account == cfg.username or\n account == '%s@%s' % (cfg.username, cfg.host))\n def happy(msg, refresh=True, changed=True):\n if changed:\n # Fun side effect: changing the passphrase invalidates the\n # message cache\n import mailpile.mailutils.emails\n mailpile.mailutils.emails.ClearParseCache(full=True)\n\n indirect_pwd = '_SECRET_:%s:%s' % (fingerprint, time.time())\n if update_ms:\n for msid, source in config.sources.iteritems():\n if _account_matches(source):\n source.password = indirect_pwd\n if update_mr:\n for msid, route in config.routes.iteritems():\n if _account_matches(route):\n route.password = indirect_pwd\n\n self._background_save(config=True)\n\n redirect = self.data.get('redirect', [None])[0]\n if redirect:\n raise UrlRedirectException(redirect)\n\n result['op_completed'] = policy\n if refresh:\n fp, r = self._prepare_result(account=account, keyid=keyid)\n result.update(r)\n\n return self._success(msg, result)\n\n if policy == 'display':\n if fingerprint in config.passphrases:\n pwd = config.passphrases[fingerprint].get_passphrase()\n elif fingerprint in config.secrets:\n pwd = config.secrets[fingerprint].password\n else:\n return self._error(_('No password found'), result=result)\n result['stored_password'] = pwd\n return happy(_('Retrieved stored password'),\n refresh=False, changed=False)\n\n if policy == 'forget':\n if fingerprint in config.passphrases:\n del config.passphrases[fingerprint]\n if fingerprint in config.secrets:\n config.secrets[fingerprint] = {'policy': 'fail'}\n del config.secrets[fingerprint]\n return happy(_('Password forgotten!'))\n\n if policy == 'fail':\n if fingerprint in config.passphrases:\n del config.passphrases[fingerprint]\n config.secrets[fingerprint] = {'policy': policy}\n return happy(_('Password will never be stored'))\n\n if policy == 'store':\n if fingerprint in config.passphrases:\n del config.passphrases[fingerprint]\n config.secrets[fingerprint] = {\n 'password': password,\n 'policy': policy}\n return happy(_('Password remembered!'))\n\n elif policy == 'cache-only' and password:\n sps = SecurePassphraseStorage(password)\n if ttl > 0:\n sps.expiration = time.time() + ttl\n config.passphrases[fingerprint] = sps\n if fingerprint in config.secrets:\n config.secrets[fingerprint] = {'policy': 'fail'}\n del config.secrets[fingerprint]\n return happy(_('The password has been stored temporarily'))\n\n else:\n return self._error(_('Invalid password policy'), result=result)\n\n\nplugin_manager = PluginManager(builtin=True)\nplugin_manager.register_commands(Authenticate,\n DeAuthenticate,\n SetPassphrase)\n" }, { "path": "mailpile/command_cache.py", "content": "import time\n\nimport mailpile.util\nfrom mailpile.commands import Command\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.plugins import PluginManager\nfrom mailpile.util import *\nfrom mailpile.ui import Session, BackgroundInteraction\n\n\n_plugins = PluginManager(builtin=__name__)\n\n\nclass CommandCache(object):\n #\n # This is an in-memory cache of commands and results we may want to\n # refresh in the background and/or reuse.\n #\n # The way this works:\n # - Cache-able commands generate a fingerprint describing themselves.\n # - If the fingerprint is found in cache, reuse the result object.\n # - Otherwise, run, generate a list of requirements and cache all of:\n # the command object itself, the requirements, the result object.\n # - Internal state changes (tag ops, new mail, etc.) call mark_dirty()\n # describing which assets (requirements) have changed.\n # - Periodically, the cache is refreshed, which re-runs any dirtied\n # commands and fires events notifying the UI about changes.\n #\n # Examples of requirements:\n #\n # - Search terms, eg. 'in:inbox' or 'potato' or 'salad'\n # - Messages: 'msg:INDEX' where INDEX is a number (not a MID)\n # - Threads: 'thread:MID' were MID is the thread ID.\n # - The app configuration: '!config'\n #\n\n def __init__(self, debug=None):\n self.debug = debug or (lambda s: None)\n self.lock = UiRLock()\n self._lag = 0.1\n self.cache = {} # id -> [exp, req, ss, cmd_obj, res_obj, added]\n self.dirty = [] # (ts, req): Requirements that changed & when\n self._dirty_ttl = 10\n\n def cache_result(self, fprint, expires, req, cmd_obj, result_obj):\n with self.lock:\n # Make a snapshot of the session, as it provides context\n ui = BackgroundInteraction(cmd_obj.session.config,\n log_parent=cmd_obj.session.ui)\n ss = Session.Snapshot(cmd_obj.session, ui=ui)\n\n # Note: We cache this even if the requirements are \"dirty\",\n # as mere presence in the cache makes this a candidate\n # for refreshing.\n self.cache[str(fprint)] = [expires, req, ss, cmd_obj, result_obj,\n time.time()]\n self.debug('Cached %s, req=%s' % (fprint, sorted(list(req))))\n\n def get_result(self, fprint, dirty_check=True, extend=300):\n with self.lock:\n exp, req, ss, co, result_obj, a = match = self.cache[fprint]\n if dirty_check:\n recent = (a > time.time() - self._lag)\n dirty = (req & self.dirty_set(after=a))\n if recent or dirty:\n # If item is too new, or requirements are dirty, pretend this\n # item does not exist.\n self.debug('Suppressing cache result %s, recent=%s dirty=%s'\n % (fprint, recent, sorted(list(dirty))))\n raise KeyError(fprint)\n match[0] = time.time() + extend\n co.session = result_obj.session = ss\n self.debug('Returning cached result for %s' % fprint)\n return result_obj\n\n def dirty_set(self, after=0):\n dirty = set(['!timedout'])\n with self.lock:\n for ts, req in self.dirty:\n if (after == 0) or (ts > after):\n dirty |= req\n return dirty\n\n def mark_dirty(self, requirements):\n with self.lock:\n self.dirty.append((time.time(), set(requirements)))\n self.debug('Marked dirty: %s' % sorted(list(requirements)))\n\n def refresh(self, extend=0, runtime=5, event_log=None):\n if mailpile.util.LIVE_USER_ACTIVITIES > 0:\n self.debug('Skipping cache refresh, user is waiting.')\n return\n\n started = now = time.time()\n with self.lock:\n # Expire things from the cache\n expired = set([f for f in self.cache if self.cache[f][0] < now])\n for fp in expired:\n del self.cache[fp]\n\n # Expire things from the dirty set\n self.dirty = [(ts, req) for ts, req in self.dirty\n if ts >= (now - self._dirty_ttl)]\n\n # Decide which fingerprints to look at this time around\n fingerprints = list(self.cache.keys())\n\n refreshed = []\n fingerprints.sort(key=lambda k: -self.cache[k][0])\n for fprint in fingerprints:\n req = None\n try:\n e, req, ss, co, ro, a = self.cache[fprint]\n now = time.time()\n dirty = (req & self.dirty_set(after=a))\n if (a + self._lag < now) and dirty:\n if now < started + runtime:\n play_nice_with_threads()\n co.session = ro.session = ss\n ro = co.refresh()\n if extend > 0:\n e = min(e + extend, now + 5*extend)\n if '!timedout' in req:\n req.remove('!timedout')\n with self.lock:\n # Make sure we do not overwrite new results from\n # elsewhere at this time.\n if self.cache[fprint][-1] == a:\n e = max(e, self.cache[fprint][0]) # Clobber?\n self.cache[fprint] = [e, req, ss, co, ro, now]\n refreshed.append(fprint)\n else:\n # Out of time, mark as dirty.\n req.add('!timedout')\n except (ValueError, IndexError, TypeError):\n # Treat broken things as if they had timed out\n if req:\n req.add('!timedout')\n\n if refreshed and event_log:\n event_log.log(message=_('New results are available'),\n source=self,\n data={'cache_ids': refreshed},\n flags=Event.COMPLETE)\n self.debug('Refreshed: %s' % refreshed)\n\n\nclass Cached(Command):\n \"\"\"Fetch results from the command cache.\"\"\"\n SYNOPSIS = (None, 'cached', 'cached', '[]')\n ORDER = ('Internals', 7)\n HTTP_QUERY_VARS = {'id': 'Cache ID of command to redisplay'}\n IS_USER_ACTIVITY = False\n LOG_NOTHING = True\n\n def max_age(self):\n # Allow result to be cached by the browser for 2 seconds; we do\n # this to facilitate cross-tab sharing of cache results.\n return 2\n\n # Warning: This depends on internals of Command, how things are run there.\n def run(self):\n try:\n cid = self.args[0] if self.args else self.data.get('id', [None])[0]\n rv = self.session.config.command_cache.get_result(cid)\n self.session.copy(rv.session)\n rv.session.ui.render_mode = self.session.ui.render_mode\n return rv\n except:\n self._starting()\n self._error(self.FAILURE % {'name': self.name,\n 'args': ' '.join(self.args)})\n return self._finishing(False)\n\n\n_plugins.register_commands(Cached)\n" }, { "path": "mailpile/commands.py", "content": "# The basic Mailpile command framework.\n#\n# TODO: Merge with plugins/ the division is obsolete and artificial.\n#\nimport json\nimport os\nimport re\nimport shlex\nimport traceback\nimport time\n\nimport mailpile.util\nimport mailpile.security as security\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.util import *\nfrom mailpile.vfs import vfs\n\n\n# Commands starting with _ don't get single-letter shortcodes...\nCOMMANDS = []\nCOMMAND_GROUPS = ['Internals', 'Config', 'Searching', 'Tagging', 'Composing']\n\n\nclass Command(object):\n \"\"\"Generic command object all others inherit from\"\"\"\n SYNOPSIS = (None, # CLI shortcode, e.g. A:\n None, # CLI shortname, e.g. add\n None, # API endpoint, e.g. sys/addmailbox\n None) # Positional argument list\n SYNOPSIS_ARGS = None # New-style positional argument list\n API_VERSION = None\n UI_CONTEXT = None\n IS_USER_ACTIVITY = False\n IS_HANGING_ACTIVITY = False\n IS_INTERACTIVE = False\n CONFIG_REQUIRED = True\n\n COMMAND_CACHE_TTL = 0 # < 1 = Not cached\n CHANGES_SESSION_CONTEXT = False\n\n FAILURE = 'Failed: %(name)s %(args)s'\n ORDER = (None, 0)\n SPLIT_ARG = True # Uses shlex by default\n RAISES = (UsageError, UrlRedirectException)\n WITH_CONTEXT = ()\n COMMAND_SECURITY = None\n\n # Event logging settings\n LOG_NOTHING = False\n LOG_ARGUMENTS = True\n LOG_PROGRESS = False\n LOG_STARTING = '%(name)s: Starting'\n LOG_FINISHED = '%(name)s: %(message)s'\n\n # HTTP settings (note: security!)\n HTTP_CALLABLE = ('GET', )\n HTTP_POST_VARS = {}\n HTTP_QUERY_VARS = {}\n HTTP_BANNED_VARS = {}\n HTTP_STRICT_VARS = True\n HTTP_AUTH_REQUIRED = True\n\n class CommandResult:\n def __init__(self, command_obj, session,\n command_name, doc, result, status, message,\n template_id=None, kwargs={}, error_info={}):\n self.session = session\n self.command_obj = command_obj\n self.command_name = command_name\n self.kwargs = {}\n self.kwargs.update(kwargs)\n self.template_id = template_id\n self.doc = doc\n self.result = result\n self.status = status\n self.error_info = {}\n self.error_info.update(error_info)\n self.message = message\n self.rendered = {}\n self.renderers = {\n 'json': self.as_json,\n 'html': self.as_html,\n 'text': self.as_text,\n 'css': self.as_css,\n 'csv': self.as_csv,\n 'rss': self.as_rss,\n 'xml': self.as_xml,\n 'txt': self.as_txt,\n 'js': self.as_js\n }\n\n def __nonzero__(self):\n return (self.result and True or False)\n\n def as_(self, what, *args, **kwargs):\n if args or kwargs:\n # Args render things un-cacheable.\n return self.renderers.get(what)(*args, **kwargs)\n\n if what not in self.rendered:\n self.rendered[what] = self.renderers.get(what, self.as_text)()\n return self.rendered[what]\n\n def as_text(self):\n if isinstance(self.result, bool):\n happy = '%s: %s' % (self.result and _('OK') or _('Failed'),\n self.message or self.doc)\n if not self.result and self.error_info:\n return '%s\\n%s' % (happy,\n json.dumps(self.error_info, indent=4,\n default=mailpile.util.json_helper))\n else:\n return happy\n elif isinstance(self.result, (dict, list, tuple)):\n return json.dumps(self.result, indent=4, sort_keys=True,\n default=mailpile.util.json_helper)\n else:\n return unicode(self.result)\n\n __str__ = lambda self: self.as_text()\n\n __unicode__ = lambda self: self.as_text()\n\n def as_dict(self):\n from mailpile.urlmap import UrlMap\n um = UrlMap(self.session)\n rv = {\n 'command': self.command_name,\n 'state': {\n 'command_url': um.ui_url(self.command_obj),\n 'context_url': um.context_url(self.command_obj),\n 'query_args': self.command_obj.state_as_query_args(),\n 'cache_id': self.command_obj.cache_id(),\n 'context': self.command_obj.context or ''\n },\n 'status': self.status,\n 'message': self.message,\n 'result': self.result,\n 'event_id': self.command_obj.event.event_id,\n 'elapsed': '%.3f' % self.session.ui.time_elapsed,\n }\n csrf_token = self.session.ui.html_variables.get('csrf_token')\n if csrf_token:\n rv['state']['csrf_token'] = csrf_token\n if self.error_info:\n rv['error'] = self.error_info\n for ui_key in [k for k in self.command_obj.data.keys()\n if k.startswith('ui_')]:\n rv[ui_key] = self.command_obj.data[ui_key][0]\n ev = self.command_obj.event\n if ev and ev.data.get('password_needed'):\n rv['password_needed'] = ev.private_data['password_needed']\n return rv\n\n def as_csv(self, template=None, result=None):\n result = self.result if (result is None) else result\n if (isinstance(result, (list, tuple)) and\n (not result or isinstance(result[0], (list, tuple)))):\n import csv, StringIO\n output = StringIO.StringIO()\n writer = csv.writer(output, dialect='excel')\n for row in result:\n writer.writerow([unicode(r).encode('utf-8') for r in row])\n return output.getvalue().decode('utf-8')\n else:\n return ''\n\n def as_json(self):\n return self.session.ui.render_json(self.as_dict())\n\n def as_html(self, template=None):\n return self.as_template('html', template)\n\n def as_js(self, template=None):\n return self.as_template('js', template)\n\n def as_css(self, template=None):\n return self.as_template('css', template)\n\n def as_rss(self, template=None):\n return self.as_template('rss', template)\n\n def as_xml(self, template=None):\n return self.as_template('xml', template)\n\n def as_txt(self, template=None):\n return self.as_template('txt', template)\n\n def as_template(self, ttype,\n mode=None, wrap_in_json=False, template=None):\n cache_id = ''.join(('j' if wrap_in_json else '', ttype,\n '/' if template else '', template or '',\n ':', mode or 'full'))\n if cache_id in self.rendered:\n return self.rendered[cache_id]\n tpath = self.command_obj.template_path(\n ttype, template_id=self.template_id, template=template)\n\n data = self.as_dict()\n data['title'] = self.message\n data['render_mode'] = mode or 'full'\n data['render_template'] = template or 'index'\n\n rendering = self.session.ui.render_web(self.session.config,\n [tpath], data)\n if wrap_in_json:\n data['result'] = rendering\n self.rendered[cache_id] = self.session.ui.render_json(data)\n else:\n self.rendered[cache_id] = rendering\n\n return self.rendered[cache_id]\n\n def __init__(self, session, name=None, arg=None, data=None, async=False):\n self.session = session\n self.context = None\n self.name = self.SYNOPSIS[1] or self.SYNOPSIS[2] or name\n self.data = data or {}\n self.status = 'unknown'\n self.message = name\n self.error_info = {}\n self.result = None\n self.run_async = async\n if type(arg) in (type(list()), type(tuple())):\n self.args = tuple(arg)\n elif arg:\n if self.SPLIT_ARG is True:\n try:\n self.args = tuple([a.decode('utf-8') for a in\n shlex.split(arg.encode('utf-8'))])\n except (ValueError, UnicodeEncodeError, UnicodeDecodeError):\n raise UsageError(_('Failed to parse arguments'))\n else:\n self.args = (arg, )\n else:\n self.args = tuple([])\n if 'arg' in self.data:\n self.args = tuple(list(self.args) + self.data['arg'])\n self._create_event()\n\n def state_as_query_args(self):\n args = {}\n if self.args:\n args['arg'] = self._sloppy_copy(self.args)\n args.update(self._sloppy_copy(self.data))\n return args\n\n def cache_id(self, sqa=None):\n if self.COMMAND_CACHE_TTL < 1:\n return ''\n from mailpile.urlmap import UrlMap\n args = sorted(list((sqa or self.state_as_query_args()).iteritems()))\n args += '/%d' % self.session.ui.term.max_width\n # The replace() stuff makes these usable as CSS class IDs\n return ('%s-%s' % (UrlMap(self.session).ui_url(self),\n md5_hex(str(args))\n )).replace('/', '-').replace('.', '-')\n\n def cache_requirements(self, result):\n raise NotImplementedError('Cachable commands should override this, '\n 'returning a set() of requirements.')\n\n def cache_result(self, result):\n if self.COMMAND_CACHE_TTL > 0:\n try:\n cache_id = self.cache_id()\n if cache_id:\n self.session.config.command_cache.cache_result(\n cache_id,\n time.time() + self.COMMAND_CACHE_TTL,\n self.cache_requirements(result),\n self,\n result)\n self.session.ui.mark(_('Cached result as %s') % cache_id)\n except (ValueError, KeyError, TypeError, AttributeError):\n self._ignore_exception()\n\n def template_path(self, ttype, template_id=None, template=None):\n path_parts = (template_id or self.SYNOPSIS[2] or 'command').split('/')\n if template in (None, ttype, 'as.' + ttype):\n path_parts.append('index')\n else:\n # Security: The template request may come from the URL, so we\n # sanitize it very aggressively before heading off\n # to the filesystem.\n clean_tpl = CleanText(template.replace('.%s' % ttype, ''),\n banned=(CleanText.FS +\n CleanText.WHITESPACE))\n path_parts.append(clean_tpl.clean)\n path_parts[-1] += '.' + ttype\n return os.path.join(*path_parts)\n\n def _gnupg(self, **kwargs):\n return GnuPG(self.session.config, event=self.event, **kwargs)\n\n def _config(self):\n session, config = self.session, self.session.config\n if not config.loaded_config:\n config.load(session)\n parent = session\n config.prepare_workers(session, daemons=self.IS_INTERACTIVE)\n if self.IS_INTERACTIVE and not config.daemons_started():\n config.prepare_workers(session, daemons=True)\n return config\n\n def _idx(self, reset=False, wait=True, wait_all=True, quiet=False):\n session, config = self.session, self._config()\n\n if not reset and config.index:\n return config.index\n\n def __do_load2():\n config.vcards.load_vcards(session)\n if not wait_all:\n session.ui.report_marks(quiet=quiet)\n\n def __do_load1():\n with config.interruptable_wait_for_lock():\n if reset:\n config.index = None\n session.results = []\n session.searched = []\n session.displayed = None\n idx = config.get_index(session)\n if wait_all:\n __do_load2()\n if not wait:\n session.ui.report_marks(quiet=quiet)\n return idx\n\n if wait:\n rv = __do_load1()\n session.ui.reset_marks(quiet=quiet)\n else:\n config.save_worker.add_task(session, 'Load', __do_load1)\n rv = None\n\n if not wait_all:\n config.save_worker.add_task(session, 'Load2', __do_load2)\n\n return rv\n\n def _background_save(self,\n everything=False, config=False,\n index=False, index_full=False,\n wait=False, wait_callback=None):\n session, cfg = self.session, self.session.config\n aut = cfg.save_worker.add_unique_task\n if everything or config:\n aut(session,\n 'Save config',\n lambda: cfg.save(session, force=(config == '!FORCE')),\n first=True)\n if cfg.index:\n cfg.flush_mbox_cache(session, clear=False, wait=wait)\n if index_full:\n aut(session, 'Save index',\n lambda: self._idx().save(session),\n first=True)\n elif everything or index:\n aut(session, 'Save index changes',\n lambda: self._idx().save_changes(session),\n first=True)\n if wait:\n wait_callback = wait_callback or (lambda: True)\n cfg.save_worker.do(session, 'Waiting', wait_callback)\n\n def _choose_messages(self, words, allow_ephemeral=False):\n msg_ids = set()\n all_words = []\n for word in words:\n all_words.extend(word.split(','))\n for what in all_words:\n if what.lower() == 'these':\n if self.session.displayed:\n b = self.session.displayed['stats']['start'] - 1\n c = self.session.displayed['stats']['count']\n msg_ids |= set(self.session.results[b:b + c])\n else:\n self.session.ui.warning(_('No results to choose from!'))\n elif what.lower() in ('all', '!all', '=!all'):\n if self.session.results:\n msg_ids |= set(self.session.results)\n else:\n self.session.ui.warning(_('No results to choose from!'))\n elif what.startswith('='):\n try:\n msg_id = int(what.replace('=', ''), 36)\n if msg_id >= 0 and msg_id < len(self._idx().INDEX):\n msg_ids.add(msg_id)\n else:\n self.session.ui.warning((_('No such ID: %s')\n ) % (what[1:], ))\n except ValueError:\n if allow_ephemeral and '-' in what:\n msg_ids.add(what[1:])\n else:\n self.session.ui.warning(_('What message is %s?'\n ) % (what, ))\n elif '-' in what:\n try:\n b, e = what.split('-')\n msg_ids |= set(self.session.results[int(b) - 1:int(e)])\n except (ValueError, KeyError, IndexError, TypeError):\n self.session.ui.warning(_('What message is %s?'\n ) % (what, ))\n else:\n try:\n msg_ids.add(self.session.results[int(what) - 1])\n except (ValueError, KeyError, IndexError, TypeError):\n self.session.ui.warning(_('What message is %s?'\n ) % (what, ))\n return msg_ids\n\n def _error(self, message, info=None, result=None):\n self.status = 'error'\n self.message = message\n\n ui_message = _('%s error: %s') % (self.name, message)\n if info:\n self.error_info.update(info)\n details = ' '.join(['%s=%s' % (k, info[k]) for k in info])\n ui_message += ' (%s)' % details\n self.session.ui.mark(self.name)\n self.session.ui.error(ui_message)\n\n if result:\n return self.view(result)\n else:\n return False\n\n def _success(self, message, result=True):\n self.status = 'success'\n self.message = message\n\n ui_message = '%s: %s' % (self.name, message)\n self.session.ui.mark(ui_message)\n\n return self.view(result)\n\n def _read_file_or_data(self, fn):\n if fn in self.data:\n return self.data[fn]\n else:\n return vfs.open(fn, 'rb').read()\n\n def _ignore_exception(self):\n self.session.ui.debug(traceback.format_exc())\n\n def _serialize(self, name, function):\n return function()\n\n def _background(self, name, function):\n session, config = self.session, self.session.config\n return config.scan_worker.add_task(session, name, function)\n\n def _update_event_state(self, state, log=False):\n self.event.flags = state\n self.event.data['elapsed'] = int(1000 * (time.time()-self._start_time))\n\n if (log or self.LOG_PROGRESS) and not self.LOG_NOTHING:\n self.event.data['ui'] = str(self.session.ui.__class__.__name__)\n self.event.data['output'] = self.session.ui.render_mode\n if self.session.config.event_log:\n self.session.config.event_log.log_event(self.event)\n\n def _starting(self):\n self._start_time = time.time()\n self._update_event_state(Event.RUNNING)\n if self.name:\n self.session.ui.start_command(self.name, self.args, self.data)\n\n def _fmt_msg(self, message):\n return message % {'name': self.name,\n 'status': self.status or '',\n 'message': self.message or ''}\n\n def _sloppy_copy(self, data, name=None):\n if name and (name[:4] in ('pass', 'csrf') or\n 'password' in name or\n 'passphrase' in name):\n data = '(SUPPRESSED)'\n def copy_value(v):\n try:\n unicode(v).encode('utf-8')\n return unicode(v)[:1024]\n except (UnicodeEncodeError, UnicodeDecodeError):\n return '(BINARY DATA)'\n if isinstance(data, (list, tuple)):\n return [self._sloppy_copy(i, name=name) for i in data]\n elif isinstance(data, dict):\n return dict((k, self._sloppy_copy(v, name=k))\n for k, v in data.iteritems())\n else:\n return copy_value(data)\n\n def _create_event(self):\n private_data = {}\n if self.LOG_ARGUMENTS:\n if self.data:\n private_data['data'] = self._sloppy_copy(self.data)\n if self.args:\n private_data['args'] = self._sloppy_copy(self.args)\n\n self.event = self._make_command_event(private_data)\n\n def _make_command_event(self, private_data):\n return Event(source=self,\n message=self._fmt_msg(self.LOG_STARTING),\n flags=Event.INCOMPLETE,\n data={},\n private_data=private_data)\n\n def _finishing(self, rv, just_cleanup=False):\n if just_cleanup:\n self._update_finished_event()\n return rv\n\n if not self.context:\n self.context = self.session.get_context(\n update=self.CHANGES_SESSION_CONTEXT)\n\n self.session.ui.mark(_('Generating result'))\n result = self.CommandResult(self, self.session, self.name,\n self.__doc__,\n rv, self.status, self.message,\n error_info=self.error_info)\n self.cache_result(result)\n\n if not self.run_async:\n self._update_finished_event()\n self.session.last_event_id = self.event.event_id\n return result\n\n def _update_finished_event(self):\n # Update the event!\n if self.message:\n self.event.message = self.message\n if self.error_info:\n self.event.private_data['error_info'] = self.error_info\n self.event.message = self._fmt_msg(self.LOG_FINISHED)\n self._update_event_state(Event.COMPLETE, log=True)\n\n self.session.ui.mark(self.event.message)\n self.session.ui.report_marks(\n details=('timing' in self.session.config.sys.debug))\n if self.name:\n self.session.ui.finish_command(self.name)\n\n def _run_sync(self, enable_cache, *args, **kwargs):\n try:\n thread_context_push(command=self,\n event=self.event,\n session=self.session)\n self._starting()\n self._run_args = args\n self._run_kwargs = kwargs\n\n if (self.COMMAND_CACHE_TTL > 0 and\n 'http' not in self.session.config.sys.debug and\n enable_cache):\n cid = self.cache_id()\n try:\n rv = self.session.config.command_cache.get_result(cid)\n rv.session.ui = self.session.ui\n if self.CHANGES_SESSION_CONTEXT:\n self.session.copy(rv.session)\n self.session.ui.mark(_('Using pre-cached result object %s') % cid)\n self._finishing(True, just_cleanup=True)\n return rv\n except:\n pass\n\n def command(self, *args, **kwargs):\n if self.CONFIG_REQUIRED:\n if not self.session.config.loaded_config:\n return self._error(_('Please log in'))\n if mailpile.util.QUITTING:\n return self._error(_('Shutting down'))\n return self.command(*args, **kwargs)\n\n return self._finishing(command(self, *args, **kwargs))\n except self.RAISES:\n self.status = 'success'\n self._finishing(True, just_cleanup=True)\n raise\n except:\n self._ignore_exception()\n self._error(self.FAILURE % {'name': self.name,\n 'args': ' '.join(self.args)})\n return self._finishing(False)\n finally:\n thread_context_pop()\n\n def _run(self, *args, **kwargs):\n if self.run_async:\n def streetcar():\n try:\n with MultiContext(self.WITH_CONTEXT):\n rv = self._run_sync(True, *args, **kwargs).as_dict()\n self.event.private_data.update(rv)\n self._update_finished_event()\n except:\n traceback.print_exc()\n\n self._starting()\n self._update_event_state(self.event.RUNNING, log=True)\n result = Command.CommandResult(self, self.session, self.name,\n self.__doc__,\n {\"resultid\": self.event.event_id},\n \"success\",\n \"Running in background\")\n\n self.session.config.scan_worker.add_task(self.session, self.name,\n streetcar, first=True)\n return result\n\n else:\n return self._run_sync(True, *args, **kwargs)\n\n def _maybe_trigger_cache_refresh(self):\n if self.data.get('_method') == 'POST':\n def refresher():\n self.session.config.command_cache.refresh(\n event_log=self.session.config.event_log)\n self.session.config.scan_worker.add_unique_task(\n self.session, 'post-refresh', refresher, first=True)\n\n def record_user_activity(self):\n mailpile.util.LAST_USER_ACTIVITY = time.time()\n\n def run(self, *args, **kwargs):\n if self.COMMAND_SECURITY is not None:\n forbidden = security.forbid_command(self)\n if forbidden:\n return self._error(forbidden)\n\n with MultiContext(self.WITH_CONTEXT):\n if self.IS_USER_ACTIVITY:\n try:\n self.record_user_activity()\n mailpile.util.LIVE_USER_ACTIVITIES += 1\n rv = self._run(*args, **kwargs)\n self._maybe_trigger_cache_refresh()\n return rv\n finally:\n mailpile.util.LIVE_USER_ACTIVITIES -= 1\n else:\n rv = self._run(*args, **kwargs)\n self._maybe_trigger_cache_refresh()\n return rv\n\n def refresh(self):\n self._create_event()\n return self._run_sync(False, *self._run_args, **self._run_kwargs)\n\n def command(self):\n return None\n\n def etag_data(self):\n return []\n\n def max_age(self):\n return 0\n\n @classmethod\n def view(cls, result):\n return result\n\n\ndef GetCommand(name):\n match = [c for c in COMMANDS if name in c.SYNOPSIS[:3]]\n if len(match) == 1:\n return match[0]\n return None\n\n\ndef Action(session, opt, arg, data=None):\n session.ui.reset_marks(quiet=True)\n config = session.config\n\n if not opt:\n return Help(session, 'help').run()\n\n # Use the COMMANDS dict by default.\n command = GetCommand(opt)\n if command:\n return command(session, opt, arg, data=data).run()\n\n # Tags are commands\n if config.loaded_config:\n lopt = opt.lower()\n\n found = None\n for tag in config.tags.values():\n if lopt == tag.slug.lower():\n found = tag\n break\n if not found:\n for tag in config.tags.values():\n if lopt == tag.name.lower():\n found = tag\n break\n if not found:\n for tag in config.tags.values():\n if lopt == _(tag.name).lower():\n found = tag\n break\n\n if found:\n a = 'in:%s%s%s' % (found.slug, ' ' if arg else '', arg)\n return GetCommand('search')(session, opt,\n arg=a, data=data).run()\n\n # OK, give up!\n raise UsageError(_('Unknown command: %s') % opt)\n" }, { "path": "mailpile/config/__init__.py", "content": "" }, { "path": "mailpile/config/base.py", "content": "from __future__ import print_function\nimport io\nimport json\nimport os\nimport ConfigParser\nfrom urllib import quote, unquote\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.util import *\n\nimport mailpile.config.validators as validators\n\n\nclass ConfigValueError(ValueError):\n pass\n\n\ndef ConfigRule(*args):\n class _ConfigRule(list):\n def __init__(self):\n list.__init__(self, args)\n self._types = []\n return _ConfigRule()\n\n\ndef PublicConfigRule(*args):\n c = ConfigRule(*args)\n c._types.append('public')\n return c\n\n\ndef KeyConfigRule(*args):\n c = ConfigRule(*args)\n c._types.append('key')\n return c\n\n\n# FIXME: This should be enforced somehow when variables are altered.\n# Run in a context?\ndef CriticalConfigRule(*args):\n c = ConfigRule(*args)\n c._types += ['critical']\n return c\n\n\ndef ConfigPrinter(cfg, indent=''):\n rv = []\n if isinstance(cfg, dict):\n pairer = cfg.iteritems()\n else:\n pairer = enumerate(cfg)\n for key, val in pairer:\n if hasattr(val, 'rules'):\n preamble = '[%s: %s] ' % (val._NAME, val._COMMENT)\n else:\n preamble = ''\n if isinstance(val, (dict, list, tuple)):\n if isinstance(val, dict):\n b, e = '{', '}'\n else:\n b, e = '[', ']'\n rv.append(('%s: %s%s\\n%s\\n%s'\n '' % (key, preamble, b, ConfigPrinter(val, ' '), e)\n ).replace('\\n \\n', ''))\n elif isinstance(val, (str, unicode)):\n rv.append('%s: \"%s\"' % (key, val))\n else:\n rv.append('%s: %s' % (key, val))\n return indent + ',\\n'.join(rv).replace('\\n', '\\n'+indent)\n\n\nclass InvalidKeyError(ValueError):\n pass\n\n\nclass CommentedEscapedConfigParser(ConfigParser.RawConfigParser):\n \"\"\"\n This is a ConfigParser that allows embedded comments and safely escapes\n and encodes/decodes values that include funky characters.\n\n >>> cfg = u'[config/sys: Stuff]\\\\ndebug = True ; Ignored comment'\n >>> cecp = CommentedEscapedConfigParser()\n >>> cecp.readfp(io.BytesIO(cfg.encode('utf-8')))\n >>> cecp.get('config/sys: Stuff', 'debug') == 'True'\n True\n\n >>> cecp.items('config/sys: Stuff')\n [(u'debug', u'True')]\n \"\"\"\n NOT_UTF8 = '%C0' # This byte is never valid at the start of an utf-8\n # string, so we use it to mark binary data.\n SAFE = '!?: /#@<>[]()=-'\n\n def set(self, section, key, value, comment):\n key = unicode(key).encode('utf-8')\n section = unicode(section).encode('utf-8')\n\n if isinstance(value, unicode):\n value = quote(value.encode('utf-8'), safe=self.SAFE)\n elif isinstance(value, str):\n quoted = quote(value, safe=self.SAFE)\n if quoted != value:\n value = self.NOT_UTF8 + quoted\n else:\n value = quote(unicode(value).encode('utf-8'), safe=self.SAFE)\n\n if value.endswith(' '):\n value = value[:-1] + '%20'\n if comment:\n pad = ' ' * (25 - len(key) - len(value)) + ' ; '\n value = '%s%s%s' % (value, pad, comment)\n return ConfigParser.RawConfigParser.set(self, section, key, value)\n\n def _decode_value(self, value):\n if value.startswith(self.NOT_UTF8):\n return unquote(value[len(self.NOT_UTF8):])\n else:\n return unquote(value).decode('utf-8')\n\n def get(self, section, key):\n key = unicode(key).encode('utf-8')\n section = unicode(section).encode('utf-8')\n value = ConfigParser.RawConfigParser.get(self, section, key)\n return self._decode_value(value)\n\n def items(self, section):\n return [(k.decode('utf-8'), self._decode_value(i)) for k, i\n in ConfigParser.RawConfigParser.items(self, section)]\n\n\ndef _MakeCheck(pcls, name, comment, rules, write_watcher):\n class Checker(pcls):\n _NAME = name\n _RULES = rules\n _COMMENT = comment\n _WWATCHER = write_watcher\n return Checker\n\n\ndef RuledContainer(pcls):\n \"\"\"\n Factory for abstract 'container with rules' class. See ConfigDict for\n details, examples and tests.\n \"\"\"\n\n class _RuledContainer(pcls):\n RULE_COMMENT = 0\n RULE_CHECKER = 1\n # Reserved ...\n RULE_DEFAULT = -1\n RULE_CHECK_MAP = {\n bool: validators.BoolCheck,\n 'bin': validators.NotUnicode,\n 'bool': validators.BoolCheck,\n 'b36': validators.B36Check,\n 'dir': validators.DirCheck,\n 'directory': validators.DirCheck,\n 'ignore': validators.IgnoreCheck,\n 'email': validators.EmailCheck,\n 'False': False, 'false': False,\n 'file': validators.FileCheck,\n 'float': float,\n 'gpgkeyid': validators.GPGKeyCheck,\n 'hostname': validators.HostNameCheck,\n 'int': int,\n 'long': long,\n 'multiline': unicode,\n 'new file': validators.NewPathCheck,\n 'new dir': validators.NewPathCheck,\n 'new directory': validators.NewPathCheck,\n 'path': validators.PathCheck,\n str: unicode,\n 'slashslug': validators.SlashSlugCheck,\n 'slug': validators.SlugCheck,\n 'str': unicode,\n 'True': True, 'true': True,\n 'timestamp': long,\n 'unicode': unicode,\n 'url': validators.UrlCheck, # FIXME: check more than the scheme?\n 'webroot': validators.WebRootCheck\n }\n def _default_write_watcher(self, *args):\n self._changed = True\n\n _NAME = 'container'\n _RULES = None\n _COMMENT = None\n _MAGIC = True\n _WWATCHER = _default_write_watcher\n\n def __init__(self, *args, **kwargs):\n rules = kwargs.get('_rules', self._RULES or {})\n self._name = kwargs.get('_name', self._NAME)\n self._comment = kwargs.get('_comment', self._COMMENT)\n self._write_watcher = kwargs.get('_write_watcher', self._WWATCHER)\n enable_magic = kwargs.get('_magic', self._MAGIC)\n for kw in ('_rules', '_comment', '_name', '_magic', '_write_watcher'):\n if kw in kwargs:\n del kwargs[kw]\n\n pcls.__init__(self)\n self._key = self._name\n self._rules_source = rules\n self._changed = False\n self.rules = {}\n self.set_rules(rules)\n self.update(*args, **kwargs)\n\n self._magic = enable_magic # Enable the getitem/getattr magic\n\n def __str__(self):\n return json.dumps(self, sort_keys=True, indent=2)\n\n def __unicode__(self):\n return json.dumps(self, sort_keys=True, indent=2)\n\n def as_config_bytes(self, _type=None, _xtype=None):\n of = io.BytesIO()\n self.as_config(_type=_type, _xtype=_xtype).write(of)\n return of.getvalue()\n\n def key_types(self, key):\n if key not in self.rules:\n key = '_any'\n if key in self.rules and hasattr(self.rules[key], '_types'):\n return self.rules[key]._types\n else:\n return []\n\n def as_config(self, config=None, _type=None, _xtype=None):\n config = config or CommentedEscapedConfigParser()\n section = self._name\n if self._comment:\n section += ': %s' % self._comment\n added_section = False\n\n keys = self.rules.keys()\n if _type:\n keys = [k for k in keys if _type in self.key_types(k)]\n\n ignore = self.ignored_keys() | set(['_any'])\n if not _type:\n if not keys or '_any' in keys:\n keys.extend(self.keys())\n\n keys = [k for k in sorted(set(keys)) if k not in ignore]\n set_keys = set(self.keys())\n\n for key in keys:\n if not hasattr(self[key], 'as_config'):\n if key in self.rules:\n comment = self.rules[key][self.RULE_COMMENT]\n else:\n comment = ''\n value = self[key]\n if value is not None and value != '':\n if key not in set_keys:\n key = ';' + key\n comment = '(default) ' + comment\n if not added_section:\n config.add_section(str(section))\n added_section = True\n if _xtype not in self.key_types(key) or not _xtype:\n config.set(section, key, value, comment)\n for key in keys:\n if hasattr(self[key], 'as_config'):\n if isinstance(self[key], list):\n # If a list is marked public, we export all items\n self[key].as_config(config=config)\n else:\n self[key].as_config(\n config=config, _type=_type, _xtype=_xtype)\n\n return config\n\n def reset(self, rules=True, data=True):\n raise Exception(_('Please override this method'))\n\n def set_rules(self, rules):\n safe_assert(isinstance(rules, dict))\n self.reset()\n for key, rule in rules.iteritems():\n self.add_rule(key, rule)\n\n def add_rule(self, key, rule):\n if not ((isinstance(rule, (list, tuple))) and\n (key == CleanText(key, banned=CleanText.NONVARS).clean) and\n (not self.real_hasattr(key))):\n raise TypeError('add_rule(%s, %s): Bad key or rule.'\n % (key, rule))\n\n orule, rule = rule, ConfigRule(*rule[:])\n if hasattr(orule, '_types'):\n rule._types = orule._types\n\n self.rules[key] = rule\n check = rule[self.RULE_CHECKER]\n try:\n check = self.RULE_CHECK_MAP.get(check, check)\n rule[self.RULE_CHECKER] = check\n except TypeError:\n pass\n\n name = '%s/%s' % (self._name, key)\n comment = rule[self.RULE_COMMENT]\n value = rule[self.RULE_DEFAULT]\n ww = self.real_getattr('_write_watcher')\n\n if (isinstance(check, dict) and value is not None\n and not isinstance(value, (dict, list))):\n raise TypeError(_('Only lists or dictionaries can contain '\n 'dictionary values (key %s).') % name)\n\n if isinstance(value, dict) and check is False:\n pcls.__setitem__(self, key, ConfigDict(_name=name,\n _comment=comment,\n _write_watcher=ww,\n _rules=value))\n\n elif isinstance(value, dict):\n if value:\n raise ConfigValueError(_('Subsections must be immutable '\n '(key %s).') % name)\n sub_rule = {'_any': [rule[self.RULE_COMMENT], check, None]}\n checker = _MakeCheck(ConfigDict, name, check, sub_rule, ww)\n pcls.__setitem__(self, key, checker())\n rule[self.RULE_CHECKER] = checker\n\n elif isinstance(value, list):\n if value:\n raise ConfigValueError(_('Lists cannot have default '\n 'values (key %s).') % name)\n sub_rule = {'_any': [rule[self.RULE_COMMENT], check, None]}\n checker = _MakeCheck(ConfigList, name, comment, sub_rule, ww)\n pcls.__setitem__(self, key, checker())\n rule[self.RULE_CHECKER] = checker\n\n elif not isinstance(value, (type(None), int, long, bool,\n float, str, unicode)):\n raise TypeError(_('Invalid type \"%s\" for key \"%s\" (value: %s)'\n ) % (type(value), name, repr(value)))\n\n def __fixkey__(self, key):\n return key.lower()\n\n def fmt_key(self, key):\n return key.lower()\n\n def get_rule(self, key):\n key = self.__fixkey__(key)\n rule = self.rules.get(key, None)\n if rule is None:\n if '_any' in self.rules:\n rule = self.rules['_any']\n else:\n raise InvalidKeyError(_('Invalid key for %s: %s'\n ) % (self._name, key))\n if isinstance(rule[self.RULE_CHECKER], dict):\n rule = rule[:]\n rule[self.RULE_CHECKER] = _MakeCheck(\n ConfigDict,\n '%s/%s' % (self._name, key),\n rule[self.RULE_COMMENT],\n rule[self.RULE_CHECKER],\n self._write_watcher)\n return rule\n\n def ignored_keys(self):\n return set([k for k in self.rules\n if self.rules[k][self.RULE_CHECKER] == validators.IgnoreCheck])\n\n def walk(self, path, parent=0, key_types=None):\n if '.' in path:\n sep = '.'\n else:\n sep = '/'\n path_parts = path.split(sep)\n cfg = self\n if parent:\n vlist = path_parts[-parent:]\n path_parts[-parent:] = []\n else:\n vlist = []\n for part in path_parts:\n if key_types is not None:\n if [t for t in cfg.key_types(part) if t not in key_types]:\n raise AccessError(_('Access denied to %s') % part)\n cfg = cfg[part]\n if parent:\n return tuple([cfg] + vlist)\n else:\n return cfg\n\n def get(self, key, default=None):\n key = self.__fixkey__(key)\n if key in self:\n return pcls.__getitem__(self, key)\n if default is None and key in self.rules:\n return self.rules[key][self.RULE_DEFAULT]\n return default\n\n def __getitem__(self, key):\n key = self.__fixkey__(key)\n if key in self.rules or '_any' in self.rules:\n return self.get(key)\n return pcls.__getitem__(self, key)\n\n def real_getattr(self, attr):\n try:\n return pcls.__getattribute__(self, attr)\n except AttributeError:\n return False\n\n def real_hasattr(self, attr):\n try:\n pcls.__getattribute__(self, attr)\n return True\n except AttributeError:\n return False\n\n def real_setattr(self, attr, value):\n return pcls.__setattr__(self, attr, value)\n\n def __getattr__(self, attr, default=None):\n if self.real_hasattr(attr) or not self.real_getattr('_magic'):\n return pcls.__getattribute__(self, attr)\n return self[attr]\n\n def __setattr__(self, attr, value):\n if self.real_hasattr(attr) or not self.real_getattr('_magic'):\n return self.real_setattr(attr, value)\n self.__setitem__(attr, value)\n\n def __passkey__(self, key, value):\n if hasattr(value, '__passkey__'):\n value._key = key\n value._name = '%s/%s' % (self._name, key)\n\n def __passkey_recurse__(self, key, value):\n if hasattr(value, '__passkey__'):\n if isinstance(value, (list, tuple)):\n for k in range(0, len(value)):\n value.__passkey__(value.__fixkey__(k), value[k])\n elif isinstance(value, dict):\n for k in value:\n value.__passkey__(value.__fixkey__(k), value[k])\n\n def __createkey_and_setitem__(self, key, value):\n pcls.__setitem__(self, key, value)\n\n def __setitem__(self, key, value):\n key = self.__fixkey__(key)\n checker = self.get_rule(key)[self.RULE_CHECKER]\n if not checker is True:\n if checker is False:\n if isinstance(value, dict) and isinstance(self[key], dict):\n for k, v in value.iteritems():\n self[key][k] = v\n return\n raise ConfigValueError(_('Modifying %s/%s is not '\n 'allowed') % (self._name, key))\n elif isinstance(checker, (list, set, tuple)):\n if value not in checker:\n raise ConfigValueError(_('Invalid value for %s/%s: %s'\n ) % (self._name, key, value))\n elif isinstance(checker, (type, type(RuledContainer))):\n try:\n if value is None:\n value = checker()\n else:\n value = checker(value)\n except (ConfigValueError):\n raise\n except (validators.IgnoreValue):\n return\n except (ValueError, TypeError):\n raise ValueError(_('Invalid value for %s/%s: %s'\n ) % (self._name, key, value))\n else:\n raise Exception(_('Unknown constraint for %s/%s: %s'\n ) % (self._name, key, checker))\n\n write_watcher = self.real_getattr('_write_watcher')\n if write_watcher is not None:\n write_watcher(self, key, value)\n\n self.__passkey__(key, value)\n self.__createkey_and_setitem__(key, value)\n self.__passkey_recurse__(key, value)\n\n def extend(self, src):\n for val in src:\n self.append(val)\n\n def __iadd__(self, src):\n self.extend(src)\n return self\n\n return _RuledContainer\n\n\nclass ConfigList(RuledContainer(list)):\n \"\"\"\n A sanity-checking, self-documenting list of program settings.\n\n Instances of this class are usually contained within a ConfigDict.\n\n >>> lst = ConfigList(_rules={'_any': ['We only like ints', int, 0]})\n >>> lst.append('1')\n '0'\n >>> lst.extend([2, '3'])\n >>> lst\n [1, 2, 3]\n\n >>> lst += ['1', '2']\n >>> lst\n [1, 2, 3, 1, 2]\n\n >>> lst.extend(range(0, 100))\n >>> lst['c'] == lst[int('c', 36)]\n True\n \"\"\"\n def reset(self, rules=True, data=True):\n if rules:\n self.rules = {}\n if data:\n self[:] = []\n\n def __createkey_and_setitem__(self, key, value):\n while key > len(self):\n self.append(self.rules['_any'][self.RULE_DEFAULT])\n if key == len(self):\n self.append(value)\n else:\n list.__setitem__(self, key, value)\n\n def append(self, value):\n list.append(self, None)\n try:\n self[len(self) - 1] = value\n return b36(len(self) - 1).lower()\n except:\n self[len(self) - 1:] = []\n raise\n\n def __passkey__(self, key, value):\n if hasattr(value, '__passkey__'):\n key = b36(key).lower()\n value._key = key\n value._name = '%s/%s' % (self._name, key)\n\n def __fixkey__(self, key):\n if isinstance(key, (str, unicode)):\n try:\n key = int(key, 36)\n except ValueError:\n pass\n return key\n\n def get(self, key, default=None):\n try:\n return list.__getitem__(self, self.__fixkey__(key))\n except IndexError:\n return default\n\n def __getitem__(self, key):\n return list.__getitem__(self, self.__fixkey__(key))\n\n def fmt_key(self, key):\n f = b36(self.__fixkey__(key)).lower()\n return ('0000' + f)[-4:] if (len(f) < 4) else f\n\n def iterkeys(self):\n return (self.fmt_key(i) for i in range(0, len(self)))\n\n def iteritems(self):\n for k in self.iterkeys():\n yield (k, self[k])\n\n def keys(self):\n return list(self.iterkeys())\n\n def all_keys(self):\n return list(self.iterkeys())\n\n def values(self):\n return self[:]\n\n def update(self, *args):\n for l in args:\n l = list(l)\n for i in range(0, len(self)):\n self[i] = l[i]\n for i in range(len(self), len(l)):\n self.append(l[i])\n\n\nclass ConfigDict(RuledContainer(dict)):\n \"\"\"\n A sanity-checking, self-documenting dictionary of program settings.\n\n The object must be initialized with a dictionary which describes in\n a structured way what variables exist, what their legal values are,\n and what their defaults are and what they are for.\n\n Each variable definition expects three values:\n 1. A human readable description of what the variable is\n 2. A data type / sanity check\n 3. A default value\n\n If the sanity check is itself a dictionary of rules, values are expected\n to be dictionaries or lists of items that match the rules defined. This\n should be used with an empty list or dictionary as a default value.\n\n Configuration data can be nested by including a dictionary of further\n rules in place of the default value.\n\n If the default value is an empty list, it is assumed to be a list of\n values of the type specified.\n\n Examples:\n\n >>> pot = ConfigDict(_rules={'potatoes': ['How many potatoes?', 'int', 0],\n ... 'carrots': ['How many carrots?', int, 99],\n ... 'liquids': ['Fluids we like', False, {\n ... 'water': ['Liters', int, 0],\n ... 'vodka': ['Liters', int, 12]\n ... }],\n ... 'tags': ['Tags', {'c': ['C', int, 0],\n ... 'x': ['X', str, '']}, []],\n ... 'colors': ['Colors', ('red', 'blue'), []]})\n >>> sorted(pot.keys()), sorted(pot.values())\n (['colors', 'liquids', 'tags'], [[], [], {}])\n\n >>> pot['potatoes'] = pot['liquids']['vodka'] = \"123\"\n >>> pot['potatoes']\n 123\n >>> pot['liquids']['vodka']\n 123\n >>> pot['carrots']\n 99\n\n >>> pot.walk('liquids.vodka')\n 123\n >>> pot.walk('liquids/vodka', parent=True)\n ({...}, 'vodka')\n\n >>> pot['colors'].append('red')\n '0'\n >>> pot['colors'].extend(['blue', 'red', 'red'])\n >>> pot['colors']\n ['red', 'blue', 'red', 'red']\n\n >>> pot['tags'].append({'c': '123', 'x': 'woots'})\n '0'\n >>> pot['tags'][0]['c']\n 123\n >>> pot['tags'].append({'z': 'invalid'})\n Traceback (most recent call last):\n ...\n ValueError: Invalid value for config/tags/1: ...\n\n >>> pot['evil'] = 123\n Traceback (most recent call last):\n ...\n InvalidKeyError: Invalid key for config: evil\n >>> pot['liquids']['evil'] = 123\n Traceback (most recent call last):\n ...\n InvalidKeyError: Invalid key for config/liquids: evil\n >>> pot['potatoes'] = \"moo\"\n Traceback (most recent call last):\n ...\n ValueError: Invalid value for config/potatoes: moo\n >>> pot['colors'].append('green')\n Traceback (most recent call last):\n ...\n ConfigValueError: Invalid value for config/colors/4: green\n\n >>> pot.rules['potatoes']\n ['How many potatoes?', , 0]\n\n >>> isinstance(pot['liquids'], ConfigDict)\n True\n \"\"\"\n _NAME = 'config'\n\n def reset(self, rules=True, data=True):\n if rules:\n self.rules = {}\n if data:\n for key in self.keys():\n if hasattr(self[key], 'reset'):\n self[key].reset(rules=rules, data=data)\n else:\n dict.__delitem__(self, key)\n\n def all_keys(self):\n return list(set(self.keys()) | set(self.rules.keys())\n - self.ignored_keys() - set(['_any']))\n\n def append(self, value):\n \"\"\"Add to the dict using an autoselected key\"\"\"\n if '_any' in self.rules:\n k = b36(max([int(k, 36) for k in self.keys()] + [-1]) + 1).lower()\n self[k] = value\n return k\n else:\n raise UsageError(_('Cannot append to fixed dict'))\n\n def update(self, *args, **kwargs):\n \"\"\"Reimplement update, so it goes through our sanity checks.\"\"\"\n for src in args:\n if hasattr(src, 'keys'):\n for key in src:\n self[key] = src[key]\n else:\n for key, val in src:\n self[key] = val\n for key in kwargs:\n self[key] = kwargs[key]\n\n def parse_config(self, session, data, source='internal'):\n \"\"\"\n Parse a config file fragment. Invalid data will be ignored, but will\n generate warnings in the session UI. Returns True on a clean parse,\n False if any of the settings were bogus.\n\n >>> cfg.parse_config(session, '[config/sys]\\\\nfd_cache_size = 123\\\\n')\n True\n >>> cfg.sys.fd_cache_size\n 123\n\n >>> cfg.parse_config(session, '[config/bogus]\\\\nblabla = bla\\\\n')\n False\n >>> [l[1] for l in session.ui.log_buffer if 'bogus' in l[1]][0]\n 'Invalid (internal): section config/bogus does not exist'\n\n >>> cfg.parse_config(session, '[config/sys]\\\\nhistory_length = 321\\\\n'\n ... 'bogus_variable = 456\\\\n')\n False\n >>> cfg.sys.history_length\n 321\n >>> [l[1] for l in session.ui.log_buffer if 'bogus_var' in l[1]][0]\n u'Invalid (internal): section config/sys, ...\n \"\"\"\n parser = CommentedEscapedConfigParser()\n parser.readfp(io.BytesIO(str(data)))\n\n def item_sorter(i):\n try:\n return (int(i[0], 36), i[1])\n except (ValueError, IndexError, KeyError, TypeError):\n return i\n\n all_okay = True\n for section in parser.sections():\n okay = True\n cfgpath = section.split(':')[0].split('/')[1:]\n cfg = self\n added_parts = []\n for part in cfgpath:\n if cfg.fmt_key(part) in cfg.keys():\n cfg = cfg[part]\n elif '_any' in cfg.rules:\n cfg[part] = {}\n cfg = cfg[part]\n else:\n if session:\n msg = _('Invalid (%s): section %s does not '\n 'exist') % (source, section)\n session.ui.warning(msg)\n all_okay = okay = False\n items = parser.items(section) if okay else []\n items.sort(key=item_sorter)\n for var, val in items:\n try:\n cfg[var] = val\n except (ValueError, KeyError, IndexError):\n if session:\n msg = _(u'Invalid (%s): section %s, variable %s=%s'\n ) % (source, section, var, val)\n session.ui.warning(msg)\n all_okay = okay = False\n return all_okay\n\n\nclass PathDict(ConfigDict):\n _RULES = {\n '_any': ['Data directory', 'directory', '']\n }\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n import copy\n\n import mailpile.config.defaults\n import mailpile.ui\n\n rules = copy.deepcopy(mailpile.config.defaults.CONFIG_RULES)\n rules.update({\n 'nest1': ['Nest1', {\n 'nest2': ['Nest2', str, []],\n 'nest3': ['Nest3', {\n 'nest4': ['Nest4', str, []]\n }, []],\n }, {}]\n })\n cfg = ConfigDict(_rules=rules)\n session = mailpile.ui.Session(cfg)\n session.ui = mailpile.ui.SilentInteraction(cfg)\n session.ui.block()\n\n result = doctest.testmod(optionflags=doctest.ELLIPSIS)\n print('%s' % (result, ))\n if result.failed:\n sys.exit(1)\n" }, { "path": "mailpile/config/defaults.py", "content": "from __future__ import print_function\nAPPVER = \"1.0.0rc6\"\nABOUT = \"\"\"\\\nMailpile.py a tool Copyright 2013-2018, Mailpile ehf\n v%8.0008s for searching and \n organizing piles of e-mail\n\nThis program is free software: you can redistribute it and/or modify it under\nthe terms of either the GNU Affero General Public License as published by the\nFree Software Foundation. See the file COPYING.md for details.\n\"\"\" % APPVER\n#############################################################################\nimport os\nimport sys\nimport time\n\nfrom mailpile.config.base import PathDict\nfrom mailpile.config.base import ConfigRule as c\nfrom mailpile.config.base import CriticalConfigRule as X\nfrom mailpile.config.base import PublicConfigRule as p\nfrom mailpile.config.base import KeyConfigRule as k\n\n\n_ = lambda string: string\n\n\nDEFAULT_SENDMAIL = '|/usr/sbin/sendmail -i %(rcpt)s'\nCONFIG_PLUGINS = []\nCONFIG_RULES = {\n 'version': p(_('Mailpile program version'), str, APPVER),\n 'homedir': p(_('Location of Mailpile data'), False, '(unset)'),\n 'timestamp': [_('Configuration timestamp'), int, int(time.time())],\n 'master_key': k(_('Master symmetric encryption key'), str, ''),\n 'sys': p(_('Technical system settings'), False, {\n 'fd_cache_size': p(_('Max files kept open at once'), int, 500),\n 'minfree_mb': p(_('Required free disk space (MB)'), int, 1024),\n 'history_length': (_('History length (lines, <0=no save)'), int, 100),\n 'http_host': p(_('Listening host for web UI'),\n 'hostname', 'localhost'),\n 'http_port': p(_('Listening port for web UI'), int, 33411),\n 'http_path': p(_('HTTP path of web UI'), 'webroot', ''),\n 'http_no_auth': X(_('Disable HTTP authentication'), bool, False),\n 'ajax_timeout': (_('AJAX Request timeout'), int, 10000),\n 'postinglist_kb': (_('Posting list target size in KB'), int, 64),\n 'sort_max': (_('Max results we sort \"well\"'), int, 2500),\n 'snippet_max': (_('Max length of metadata snippets'), int, 275),\n 'debug': p(_('Debugging flags'), str, ''),\n 'experiments': (_('Enabled experiments'), str, ''),\n 'gpg_keyserver': (_('Host:port of PGP keyserver'),\n str, 'pool.sks-keyservers.net'),\n 'gpg_home': p(_('Override the home directory of GnuPG'),\n 'dir', None),\n 'gpg_binary': p(_('Override the default GPG binary path'),\n 'file', None),\n 'local_mailbox_id': (_('Local read/write Maildir'), 'b36', ''),\n 'mailindex_file': (_('Metadata index file'), 'file', ''),\n 'postinglist_dir': (_('Search index directory'), 'dir', ''),\n 'mailbox': [_('Mailboxes we index'), 'bin', []],\n 'plugins_early': p(_('Plugins to load before login'),\n CONFIG_PLUGINS, []),\n 'plugins': [_('Plugins to load after login'),\n CONFIG_PLUGINS, []],\n 'path': [_('Locations of assorted data'), False, {\n 'html_theme': [_('User interface theme'), 'dir', 'default-theme'],\n 'vcards': [_('Location of vCards'), 'dir', 'vcards'],\n 'event_log': [_('Location of event log'), 'dir', 'logs'],\n }],\n 'lockdown': p(_('Demo mode, disallow changes'), str, ''),\n 'login_banner': p(_('A custom banner for the login page'), str, ''),\n 'proxy': p(_('Proxy settings'), False, {\n 'protocol': p(_('Proxy protocol'),\n [\"tor\", \"tor-risky\", \"socks5\", \"socks4\", \"http\",\n \"none\", \"system\", \"unknown\"], \"system\"),\n 'fallback': p(_('Allow fallback to direct conns'), bool, False),\n 'username': (_('User name'), str, ''),\n 'password': (_('Password'), str, ''),\n 'host': p(_('Host'), str, ''),\n 'port': p(_('Port'), int, 8080),\n 'no_proxy': p(_('List of hosts to avoid proxying'), str,\n 'localhost, 127.0.0.1, ::1')\n }),\n 'tor': p(_('Tor settings'), False, {\n 'binary': p(_('Override the default Tor binary path'),\n 'file', None),\n 'systemwide':p(_('Use shared system-wide Tor (not our own)'),\n bool, True),\n 'socks_host':p(_('Socks host'), str, ''),\n 'socks_port':p(_('Socks Port'), int, 0),\n 'ctrl_port': p(_('Control Port'), int, 0),\n 'ctrl_auth': p(_('Control Password'), str, '')\n })\n }),\n 'prefs': p(_(\"User preferences\"), False, {\n 'num_results': (_('Search results per page'), int, 20),\n 'rescan_interval': (_('Misc. data refresh frequency'), int, 900),\n 'open_in_browser': p(_('Open in browser on startup'), bool, True),\n 'auto_mark_as_read':(_('Automatically mark e-mail as read'),\n bool, True),\n 'web_content': (_('Download content from the web'),\n [\"off\", \"anon\", \"on\"], \"unknown\"),\n 'html5_sandbox': (_('Use HTML5 sandboxes'), bool, True),\n 'attachment_urls': (_('URLs to treat as attachments (regex)'), str, []),\n 'weak_crypto_max_age': (\n _('Accept weak crypto in messages older than this (unix time)'),\n int, 0),\n 'encrypted_block_html': (_('Never display HTML from encrypted mail'),\n bool, True),\n 'encrypted_block_web': (_('Never fetch web content from encrypted mail'),\n bool, True),\n 'gpg_use_agent': (_('Use the local GnuPG agent'), bool, False),\n 'gpg_clearsign': X(_('Inline PGP signatures or attached'),\n bool, False),\n 'gpg_recipient': (_('Encrypt local data to ...'), 'gpgkeyid', ''),\n 'gpg_email_key': (_('Enable e-mail based public key distribution'),\n bool, True),\n 'gpg_html_wrap': (_('Wrap keys and signatures in helpful HTML'),\n bool, True),\n 'antiphishing': (_(\"Enable experimental anti-phishing heuristics \"),\n bool, False),\n 'key_tofu': (_(\"Key Import Behaviour\"), False, {\n 'autocrypt': (_('Auto-import keys using Autocrypt state machine'),\n bool, True),\n 'historic': (_('Auto-import keys using communication history'),\n bool, True),\n 'hist_min': (_('Require this many signed- or encrypted e-mails'),\n int, 3),\n 'hist_recent': (_('Consider the most recent N e-mails (per sender)'),\n int, 6),\n 'hist_origins':(_('Origins to auto-import keys from (historic)'),\n str, 'e-mail, wkd, koo'),\n 'min_interval': (_('Interval between TOFU checks (per sender)'),\n int, 1800),\n }),\n 'key_trust': (_(\"Key Trust Model\"), False, {\n 'threshold': (_('Minimum number of signatures required'),\n int, 5),\n 'window_days': (_('Window of time (days) to evaluate trust'),\n int, 180),\n 'sig_warn_pct': (_('Signed ratio (%) above which we expect sigs'),\n int, 80),\n 'key_trust_pct':(_('Ratio of key use (%) above which we trust key'),\n int, 90),\n 'key_new_pct': (_('Consider key new below this ratio (%) of sigs'),\n int, 10)\n }),\n 'openpgp_header': X(_('Advertise PGP preferences in a header?'),\n ['', 'sign', 'encrypt', 'signencrypt'],\n 'signencrypt'),\n 'crypto_policy': X(_('Default encryption policy for outgoing mail'),\n str, 'none'),\n 'inline_pgp': (_('Use inline PGP when possible'), bool, True),\n 'encrypt_subject': (_('Encrypt subjects by default'), bool, True),\n 'default_order': (_('Default sort order'), str, 'rev-date'),\n 'obfuscate_index':X(_('Key to use to scramble the index'), str, ''),\n 'index_encrypted':X(_('Make encrypted content searchable'),\n bool, False),\n 'encrypt_mail': X(_('Encrypt locally stored mail'), bool, False),\n 'encrypt_index': X(_('Encrypt the local search index'), bool, False),\n 'encrypt_vcards': X(_('Encrypt the contact database'), bool, True),\n 'encrypt_events': X(_('Encrypt the event log'), bool, True),\n 'encrypt_misc': X(_('Encrypt misc. local data'), bool, True),\n 'allow_deletion': X(_('Allow permanent deletion of e-mails'),\n bool, False),\n 'deletion_ratio': X(_('Max fraction of source mail to delete per pass'),\n float, 0.75),\n# FIXME:\n# 'backup_to_web': X(_('Backup settings and keys to mobile web app'),\n# bool, True),\n# 'backup_to_email':X(_('Backup settings and keys to e-mail'), str, ''),\n 'rescan_command': (_('Command run before rescanning'), str, ''),\n 'default_email': (_('Default outgoing e-mail address'), 'email', ''),\n 'default_route': (_('Default outgoing mail route'), str, ''),\n 'line_length': (_('Target line length, <40 disables reflow'),\n int, 65),\n 'always_bcc_self': (_('Always BCC self on outgoing mail'), bool, True),\n 'default_messageroute': (_('Default outgoing mail route'), str, ''),\n 'language': p(_('User interface language'), str, ''),\n 'vcard': [_(\"vCard import/export settings\"), False, {\n 'importers': [_(\"vCard import settings\"), False, {}],\n 'exporters': [_(\"vCard export settings\"), False, {}],\n 'context': [_(\"vCard context helper settings\"), False, {}],\n }],\n 'friendly_pipes': (_(\"Enable sh-like pipes in the CLI\"), bool, True),\n }),\n 'web': (_(\"Web Interface Preferences\"), False, {\n 'keybindings': (_('Enable keyboard short-cuts'), bool, False),\n 'developer_mode': (_('Enable developer-only features'), bool, False),\n 'friendly_dates': (_('UI uses \"friendly\" date/times'), bool, True),\n 'setup_complete': (_('User completed setup experience'), bool, False),\n 'display_density': (_('Display density of interface'), str, 'comfy'),\n 'quoted_reply': (_('Quote replies to messages'), str, 'unset'),\n 'nag_backup_key': (_('Nag user to backup their key'), int, 0),\n 'subtags_collapsed': (_('Collapsed subtags in sidebar'), str, []),\n 'donate_visibility': (_('Display donate link in topbar?'), bool, True),\n 'email_html_hint': (_('Display HTML hints?'), bool, True),\n 'email_crypto_hint': (_('Display crypto hints?'), bool, True),\n 'email_reply_hint': (_('Display reply hints?'), bool, True),\n 'email_tag_hint': (_('Display tagging hints?'), bool, True),\n 'release_notes': (_('Display release notes?'), bool, True)\n }),\n 'logins': [_('Credentials allowed to access Mailpile'), {\n 'password': (_('Salted and hashed password'), str, '')\n }, {}],\n 'secrets': [_('Secrets the user wants saved'), {\n 'password': (_('A secret'), str, ''),\n 'policy': (_('Security policy'),\n [\"store\", \"cache-only\", \"fail\", \"protect\"],\n 'store')\n }, {}],\n 'tls': [_('Settings for TLS certificate validation'), {\n 'server': (_('Server hostname:port'), str, ''),\n 'accept_certs': (_('SHA256 of acceptable certs'), str, []),\n 'use_web_ca': (_('Use web certificate authorities'), bool, True)\n }, {}],\n 'routes': [_('Outgoing message routes'), {\n 'name': (_('Route name'), str, ''),\n 'protocol': (_('Messaging protocol'),\n [\"smtp\", \"smtptls\", \"smtpssl\", \"local\"],\n 'smtp'),\n 'username': (_('User name'), str, ''),\n 'password': (_('Password'), str, ''),\n 'auth_type': (_('Authentication scheme'), str, 'password-cleartext'),\n 'command': (_('Shell command'), str, ''),\n 'host': (_('Host'), str, ''),\n 'port': (_('Port'), int, 587)\n }, {}],\n 'sources': [_('Incoming message sources'), {\n 'name': (_('Source name'), str, ''),\n 'profile': (_('Profile this source belongs to'), str, ''),\n 'enabled': (_('Is this mail source enabled?'), bool, True),\n 'protocol': (_('Mail source protocol'),\n [\"local\",\n \"imap\", \"imap_ssl\", \"imap_tls\",\n \"pop3\", \"pop3_ssl\",\n # These are all obsolete, handled as local:\n \"mbox\", \"maildir\", \"macmaildir\", \"gmvault\"],\n ''),\n 'pre_command': (_('Shell command run before syncing'), str, ''),\n 'post_command': (_('Shell command run after syncing'), str, ''),\n 'interval': (_('How frequently to check for mail'), int, 300),\n 'username': (_('User name'), str, ''),\n 'password': (_('Password'), str, ''),\n 'auth_type': (_('Authentication scheme'), str, 'password'),\n 'host': (_('Host'), str, ''),\n 'port': (_('Port'), int, 993),\n 'keepalive': (_('Keep server connections alive'), bool, False),\n 'discovery': (_('Mailbox discovery policy'), False, {\n 'paths': (_('Paths to watch for new mailboxes'), 'bin', []),\n 'policy': (_('Default mailbox policy'),\n ['unknown', 'ignore', 'watch',\n 'read', 'move', 'sync'], 'unknown'),\n 'local_copy': (_('Copy mail to a local mailbox?'), bool, False),\n 'parent_tag': (_('Parent tag for mailbox tags'), str, '!CREATE'),\n 'guess_tags': (_('Guess which local tags match'), bool, True),\n 'create_tag': (_('Create a tag for each mailbox?'), bool, True),\n 'visible_tags':(_('Make tags visible by default?'), bool, True),\n 'process_new': (_('Is a potential source of new mail'), bool, True),\n 'apply_tags': (_('Tags applied to messages'), str, []),\n 'max_mailboxes':(_('Max mailboxes to add'), int, 100),\n }),\n 'mailbox': (_('Mailboxes'), {\n 'name': (_('The name of this mailbox'), str, ''),\n 'path': (_('Mailbox source path'), str, ''),\n 'policy': (_('Mailbox policy'),\n ['unknown', 'ignore', 'read', 'move', 'sync',\n 'inherit'], 'inherit'),\n 'local': (_('Local mailbox path'), 'bin', ''),\n 'process_new': (_('Is a source of new mail'), bool, True),\n 'primary_tag': (_('A tag representing this mailbox'), str, ''),\n 'apply_tags': (_('Tags applied to messages'), str, []),\n }, {})\n }, {}]\n}\n\n\nif __name__ == \"__main__\":\n import mailpile.config.defaults\n from mailpile.config.base import ConfigDict\n\n print('%s' % (ConfigDict(_name='mailpile',\n _comment='Base configuration',\n _rules=mailpile.config.defaults.CONFIG_RULES\n ).as_config_bytes(), ))\n" }, { "path": "mailpile/config/detect.py", "content": "try:\n import ssl\nexcept ImportError:\n ssl = None\n\ntry:\n import sockschain as socks\nexcept ImportError:\n try:\n import socks\n except ImportError:\n socks = None\n" }, { "path": "mailpile/config/manager.py", "content": "from __future__ import print_function\nimport copy\nimport cPickle\nimport io\nimport jinja2\nimport json\nimport os\nimport socket\nimport sys\nimport random\nimport re\nimport threading\nimport fasteners\nimport traceback\nimport ConfigParser\nimport errno\n\nfrom urllib import quote, unquote, getproxies\nfrom urlparse import urlparse\n\ntry:\n from appdirs import AppDirs\nexcept ImportError:\n AppDirs = None\n\nimport mailpile.platforms\nfrom mailpile.command_cache import CommandCache\nfrom mailpile.crypto.streamer import DecryptingStreamer\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.eventlog import EventLog, Event, GetThreadEvent\nfrom mailpile.httpd import HttpWorker\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import OpenMailbox, NoSuchMailboxError, wervd\nfrom mailpile.mailutils import FormatMbxId, MBX_ID_LEN\nfrom mailpile.search import MailIndex\nfrom mailpile.search_history import SearchHistory\nfrom mailpile.security import SecurePassphraseStorage\nfrom mailpile.ui import Session, BackgroundInteraction\nfrom mailpile.util import *\nfrom mailpile.vcard import VCardStore\nfrom mailpile.vfs import vfs, FilePath, MailpileVfsRoot\nfrom mailpile.workers import Worker, ImportantWorker, DumbWorker, Cron\nimport mailpile.i18n\nimport mailpile.security\nimport mailpile.util\nimport mailpile.vfs\n\nfrom mailpile.config.base import *\nfrom mailpile.config.paths import DEFAULT_WORKDIR, DEFAULT_SHARED_DATADIR\nfrom mailpile.config.paths import LOCK_PATHS\nfrom mailpile.config.defaults import APPVER\nfrom mailpile.config.detect import socks\nfrom mailpile.www.jinjaloader import MailpileJinjaLoader\n\n\nMAX_CACHED_MBOXES = 5\n\nGLOBAL_INDEX_CHECK = ConfigLock()\nGLOBAL_INDEX_CHECK.acquire()\n\n\nclass ConfigManager(ConfigDict):\n \"\"\"\n This class manages the live global mailpile configuration. This includes\n the settings themselves, as well as global objects like the index and\n references to any background worker threads.\n \"\"\"\n def __init__(self, workdir=None, shareddatadir=None, rules={}):\n ConfigDict.__init__(self, _rules=rules, _magic=False)\n\n self.workdir = os.path.abspath(workdir or DEFAULT_WORKDIR())\n self.gnupghome = None\n mailpile.vfs.register_alias('/Mailpile', self.workdir)\n\n self.shareddatadir = os.path.abspath(shareddatadir or\n DEFAULT_SHARED_DATADIR())\n mailpile.vfs.register_alias('/Share', self.shareddatadir)\n\n self.vfs_root = MailpileVfsRoot(self)\n mailpile.vfs.register_handler(0000, self.vfs_root)\n\n self.conffile = os.path.join(self.workdir, 'mailpile.cfg')\n self.conf_key = os.path.join(self.workdir, 'mailpile.key')\n self.conf_pub = os.path.join(self.workdir, 'mailpile.rc')\n\n # Process lock files are not actually created until the first acquire()\n self.lock_pubconf, self.lock_workdir = LOCK_PATHS(self.workdir)\n self.lock_pubconf = fasteners.InterProcessLock(self.lock_pubconf)\n\n # If the master key changes, we update the file on save, otherwise\n # the file is untouched. So we keep track of things here.\n self._master_key = ''\n self._master_key_ondisk = None\n self._master_key_passgen = -1\n self.detected_memory_corruption = False\n\n # Make sure we have a silent background session\n self.background = Session(self)\n self.background.ui = BackgroundInteraction(self)\n\n self.plugins = None\n self.tor_worker = None\n self.http_worker = None\n self.dumb_worker = DumbWorker('Dumb worker', self.background)\n self.slow_worker = self.dumb_worker\n self.scan_worker = self.dumb_worker\n self.save_worker = self.dumb_worker\n self.other_workers = []\n self.mail_sources = {}\n\n self.event_log = None\n self.index = None\n self.index_loading = None\n self.index_check = GLOBAL_INDEX_CHECK\n self.vcards = {}\n self.search_history = SearchHistory()\n self._mbox_cache = []\n self._running = {}\n self._lock = ConfigRLock()\n self.loaded_config = False\n\n def cache_debug(msg):\n if self.background and 'cache' in self.sys.debug:\n self.background.ui.debug(msg)\n self.command_cache = CommandCache(debug=cache_debug)\n\n self.passphrases = {\n 'DEFAULT': SecurePassphraseStorage(),\n }\n\n self.jinja_env = jinja2.Environment(\n loader=MailpileJinjaLoader(self),\n cache_size=400,\n autoescape=True,\n trim_blocks=True,\n extensions=['jinja2.ext.i18n', 'jinja2.ext.with_',\n 'jinja2.ext.do', 'jinja2.ext.autoescape',\n 'mailpile.www.jinjaextensions.MailpileCommand']\n )\n\n self.cron_schedule = {}\n self.cron_worker = Cron(self.cron_schedule, 'Cron worker', self.background)\n self.cron_worker.daemon = True\n self.cron_worker.start()\n\n self._magic = True # Enable the getattr/getitem magic\n\n def create_and_lock_workdir(self, session):\n # Make sure workdir exists and that other processes are not using it.\n if not os.path.exists(self.workdir):\n if session:\n session.ui.notify(_('Creating: %s') % self.workdir)\n os.makedirs(self.workdir, mode=0o700)\n mailpile.platforms.RestrictReadAccess(self.workdir)\n\n # Once acquired, lock_workdir is only released by process termination.\n if not isinstance(self.lock_workdir, fasteners.InterProcessLock):\n ipl = fasteners.InterProcessLock(self.lock_workdir)\n if ipl.acquire(blocking=False):\n self.lock_workdir = ipl\n else:\n if session:\n session.ui.error(_('Another Mailpile or program is'\n ' using the profile directory'))\n sys.exit(1)\n\n def load(self, session, *args, **kwargs):\n from mailpile.plugins.core import Rescan\n\n # This should happen somewhere, may as well happen here. We don't\n # rely on Python's random for anything important, but it's still\n # nice to seed it well.\n random.seed(os.urandom(8))\n\n keep_lockdown = self.sys.lockdown\n with self._lock:\n rv = self._unlocked_load(session, *args, **kwargs)\n\n if not kwargs.get('public_only'):\n # If the app version does not match the config, run setup.\n if self.version != APPVER:\n from mailpile.plugins.setup_magic import Setup\n Setup(session, 'setup').run()\n\n # Trigger background-loads of everything\n Rescan(session, 'rescan')._idx(wait=False)\n\n # Record where our GnuPG keys live\n self.gnupghome = GnuPG(self).gnupghome()\n\n if keep_lockdown:\n self.sys.lockdown = keep_lockdown\n return rv\n\n def load_master_key(self, passphrase, _raise=None):\n keydata = []\n\n if passphrase.is_set():\n with open(self.conf_key, 'rb') as fd:\n hdrs = dict(l.split(': ', 1) for l in fd if ': ' in l)\n salt = hdrs.get('Salt', '').strip()\n kdfp = hdrs.get('KDF', '').strip() or None\n\n if kdfp:\n try:\n kdf, params = kdfp.split(' ', 1)\n kdfp = {}\n kdfp[kdf] = json.loads(params)\n except ValueError:\n kdfp = {}\n\n parser = lambda d: keydata.extend(d)\n for (method, sps) in passphrase.stretches(salt, params=kdfp):\n try:\n with open(self.conf_key, 'rb') as fd:\n decrypt_and_parse_lines(fd, parser, self,\n newlines=True,\n gpgi=GnuPG(self),\n passphrase=sps)\n break\n except IOError:\n keydata = []\n\n if keydata:\n self.passphrases['DEFAULT'].copy(passphrase)\n self.set_master_key(''.join(keydata))\n self._master_key_ondisk = self.get_master_key()\n self._master_key_passgen = self.passphrases['DEFAULT'].generation\n return True\n else:\n if _raise is not None:\n raise _raise('Failed to decrypt master key')\n return False\n\n def _load_config_lines(self, filename, lines):\n collector = lambda ll: lines.extend(ll)\n if os.path.exists(filename):\n with open(filename, 'rb') as fd:\n decrypt_and_parse_lines(fd, collector, self)\n\n def _discover_plugins(self):\n # Discover plugins and update the config rule to match\n from mailpile.plugins import PluginManager\n self.plugins = PluginManager(config=self, builtin=True).discover([\n os.path.join(self.shareddatadir, 'contrib'),\n os.path.join(self.workdir, 'plugins')\n ])\n self.sys.plugins.rules['_any'][\n self.RULE_CHECKER] = [None] + self.plugins.loadable()\n self.sys.plugins_early.rules['_any'][\n self.RULE_CHECKER] = [None] + self.plugins.loadable_early()\n\n def _configure_default_plugins(self):\n if (len(self.sys.plugins) == 0) and self.loaded_config:\n self.sys.plugins.extend(self.plugins.DEFAULT)\n for plugin in self.plugins.WANTED:\n if plugin in self.plugins.available():\n self.sys.plugins.append(plugin)\n else:\n for pos in self.sys.plugins.keys():\n name = self.sys.plugins[pos]\n if name in self.plugins.RENAMED:\n self.sys.plugins[pos] = self.plugins.RENAMED[name]\n\n def _unlocked_load(self, session, public_only=False):\n # This method will attempt to load the full configuration.\n #\n # The Mailpile configuration is in two parts:\n # - public data in \"mailpile.rc\"\n # - private data in \"mailpile.cfg\" (encrypted)\n #\n # This method may successfully load and process from the public part,\n # but fail to load the encrypted part due to a lack of authentication.\n # In this case IOError will be raised.\n #\n if not public_only:\n self.create_and_lock_workdir(session)\n if session is None:\n session = self.background\n if self.index:\n self.index_check.acquire()\n self.index = None\n\n # Set the homedir default\n self.rules['homedir'][2] = self.workdir\n self._rules_source['homedir'][2] = self.workdir\n self.reset(rules=False, data=True)\n self.loaded_config = False\n pub_lines, prv_lines = [], []\n try:\n self._load_config_lines(self.conf_pub, pub_lines)\n if public_only:\n return\n\n if os.path.exists(self.conf_key):\n mailpile.platforms.RestrictReadAccess(self.conf_key)\n self.load_master_key(self.passphrases['DEFAULT'],\n _raise=IOError)\n self._load_config_lines(self.conffile, prv_lines)\n except IOError:\n self.loaded_config = False\n raise\n except (ValueError, OSError):\n # Bad data in config or config doesn't exist: just forge onwards\n pass\n finally:\n ## The following things happen, no matter how loading went...\n\n # Discover plugins first, as this affects what is or is not valid\n # in the configuration file.\n self._discover_plugins()\n\n # Parse once (silently), to figure out which plugins to load...\n self.parse_config(None, '\\n'.join(pub_lines), source=self.conf_pub)\n self.parse_config(None, '\\n'.join(prv_lines), source=self.conffile)\n\n # Enable translations!\n mailpile.i18n.ActivateTranslation(\n session, self, self.prefs.language)\n\n # Configure and load plugins as per config requests\n with mailpile.i18n.i18n_disabled:\n self._configure_default_plugins()\n self.load_plugins(session)\n\n # Now all the plugins are loaded, reset and parse again!\n self.reset_rules_from_source()\n self.parse_config(session, '\\n'.join(pub_lines), source=self.conf_pub)\n self.parse_config(session, '\\n'.join(prv_lines), source=self.conffile)\n self._changed = False\n\n # Do this again, so renames and cleanups persist\n self._configure_default_plugins()\n\n ## The following events only happen when we've successfully loaded\n ## both config files!\n\n # Open event log\n dec_key_func = lambda: self.get_master_key()\n enc_key_func = lambda: (self.prefs.encrypt_events and\n self.get_master_key())\n self.event_log = EventLog(self.data_directory('event_log',\n mode='rw', mkdir=True),\n dec_key_func, enc_key_func\n ).load()\n if 'log' in self.sys.debug:\n self.event_log.ui_watch(session.ui)\n else:\n self.event_log.ui_unwatch(session.ui)\n\n # Configure security module\n mailpile.security.KNOWN_TLS_HOSTS = self.tls\n\n # Load VCards\n self.vcards = VCardStore(self, self.data_directory('vcards',\n mode='rw',\n mkdir=True))\n\n # Recreate VFS root in case new things have been found\n self.vfs_root.rescan()\n\n # Load Search History\n self.search_history = SearchHistory.Load(self,\n merge=self.search_history)\n\n # OK, we're happy\n self.loaded_config = True\n\n def reset_rules_from_source(self):\n with self._lock:\n self.set_rules(self._rules_source)\n self.sys.plugins.rules['_any'][\n self.RULE_CHECKER] = [None] + self.plugins.loadable()\n self.sys.plugins_early.rules['_any'][\n self.RULE_CHECKER] = [None] + self.plugins.loadable_early()\n\n def load_plugins(self, session):\n with self._lock:\n from mailpile.plugins import PluginManager\n plugin_list = set(PluginManager.REQUIRED +\n self.sys.plugins +\n self.sys.plugins_early)\n for plugin in plugin_list:\n if plugin is not None:\n session.ui.mark(_('Loading plugin: %s') % plugin)\n self.plugins.load(plugin)\n session.ui.mark(_('Processing manifests'))\n self.plugins.process_manifests()\n self.prepare_workers(session)\n\n def save(self, *args, **kwargs):\n with self._lock:\n self._unlocked_save(*args, **kwargs)\n\n def get_master_key(self):\n if not self._master_key:\n return ''\n\n k1, k2, k3 = (k[1:] for k in self._master_key)\n if k1 == k2 == k3:\n # This is the only result we like!\n return k1\n else:\n # Hard fail into read-only lockdown. The periodic health\n # check will notify the user we are broken.\n self.detected_memory_corruption = True\n\n # Try and recover; best 2 out of 3.\n if k1 in (k2, k3):\n return self.set_master_key(k1)\n if k2 in (k1, k3):\n return self.set_master_key(k2)\n\n mailpile.util.QUITTING = True\n raise IOError(\"Failed to access master_key\")\n\n def set_master_key(self, key):\n # Prefix each key with a unique character to prevent optimization\n self._master_key = [i + key for i in ('1', '2', '3')]\n return key\n\n def _delete_old_master_keys(self, keyfile):\n \"\"\"\n We keep old master key files around for up to 5 days, so users can\n revert if they make some sort of horrible mistake. After that we\n delete the backups because they're technically a security risk.\n \"\"\"\n maxage = time.time() - (5 * 24 * 3600)\n prefix = os.path.basename(keyfile) + '.'\n dirname = os.path.dirname(keyfile)\n for f in os.listdir(dirname):\n fn = os.path.join(dirname, f)\n if f.startswith(prefix) and (os.stat(fn).st_mtime < maxage):\n safe_remove(fn)\n\n def _save_master_key(self, keyfile):\n if not self.get_master_key():\n return False\n\n # We keep the master key in a file of its own...\n want_renamed_keyfile = None\n master_passphrase = self.passphrases['DEFAULT']\n if (self._master_key_passgen != master_passphrase.generation\n or self._master_key_ondisk != self.get_master_key()):\n if os.path.exists(keyfile):\n want_renamed_keyfile = keyfile + ('.%x' % time.time())\n\n if not want_renamed_keyfile and os.path.exists(keyfile):\n # Key file exists, nothing needs to be changed. Happy!\n # Delete any old key backups we have laying around\n self._delete_old_master_keys(keyfile)\n return True\n\n # Figure out whether we are encrypting to a GPG key, or using\n # symmetric encryption (with the 'DEFAULT' passphrase).\n gpgr = self.prefs.get('gpg_recipient', '').replace(',', ' ')\n tokeys = (gpgr.split()\n if (gpgr and gpgr not in ('!CREATE', '!PASSWORD'))\n else None)\n\n if not tokeys and not master_passphrase.is_set():\n # Without recipients or a passphrase, we cannot save!\n return False\n\n if not tokeys:\n salt = b64w(os.urandom(32).encode('base64'))\n else:\n salt = ''\n\n # FIXME: Create event and capture GnuPG state?\n mps = master_passphrase.stretched(salt)\n gpg = GnuPG(self, passphrase=mps)\n status, encrypted_key = gpg.encrypt(self.get_master_key(), tokeys=tokeys)\n if status == 0:\n if salt:\n h, b = encrypted_key.replace('\\r', '').split('\\n\\n', 1)\n encrypted_key = ('%s\\nSalt: %s\\nKDF: %s\\n\\n%s'\n % (h, salt, mps.is_stretched or 'None', b))\n try:\n with open(keyfile + '.new', 'wb') as fd:\n fd.write(encrypted_key)\n mailpile.platforms.RestrictReadAccess(keyfile + '.new')\n if want_renamed_keyfile:\n os.rename(keyfile, want_renamed_keyfile)\n os.rename(keyfile + '.new', keyfile)\n self._master_key_ondisk = self.get_master_key()\n self._master_key_passgen = master_passphrase.generation\n\n # Delete any old key backups we have laying around\n self._delete_old_master_keys(keyfile)\n\n return True\n except:\n if (want_renamed_keyfile and\n os.path.exists(want_renamed_keyfile)):\n os.rename(want_renamed_keyfile, keyfile)\n raise\n\n return False\n\n def _unlocked_save(self, session=None, force=False):\n newfile = '%s.new' % self.conffile\n pubfile = self.conf_pub\n keyfile = self.conf_key\n\n self.create_and_lock_workdir(None)\n self.timestamp = int(time.time())\n\n if session and self.event_log:\n if 'log' in self.sys.debug:\n self.event_log.ui_watch(session.ui)\n else:\n self.event_log.ui_unwatch(session.ui)\n\n # Save the public config data first\n # Warn other processes against reading public data during write\n # But wait for 2 s max so other processes can't block Mailpile.\n try:\n locked = self.lock_pubconf.acquire(blocking=True, timeout=2)\n with open(pubfile, 'wb') as fd:\n fd.write(self.as_config_bytes(_type='public'))\n finally:\n if locked:\n self.lock_pubconf.release()\n if not self.loaded_config:\n return\n\n # Save the master key if necessary (and possible)\n master_key_saved = self._save_master_key(keyfile)\n\n # We abort the save here if nothing has changed.\n if not force and not self._changed:\n return\n\n # Reset our \"changed\" tracking flag. Any changes that happen\n # during the subsequent saves will mark us dirty again, since\n # we can't be sure the changes got written out.\n self._changed = False\n\n # This slight over-complication, is a reaction to segfaults in\n # Python 2.7.5's fd.write() method. Let's just feed it chunks\n # of data and hope for the best. :-/\n config_bytes = self.as_config_bytes(_xtype='public')\n config_chunks = (config_bytes[i:i + 4096]\n for i in range(0, len(config_bytes), 4096))\n\n from mailpile.crypto.streamer import EncryptingStreamer\n if self.get_master_key() and master_key_saved:\n subj = self.mailpile_path(self.conffile)\n with EncryptingStreamer(self.get_master_key(),\n dir=self.tempfile_dir(),\n header_data={'subject': subj},\n name='Config') as fd:\n for chunk in config_chunks:\n fd.write(chunk)\n fd.save(newfile)\n else:\n # This may result in us writing the master key out in the\n # clear, but that is better than losing data. :-(\n with open(newfile, 'wb') as fd:\n for chunk in config_chunks:\n fd.write(chunk)\n\n # Keep the last 5 config files around... just in case.\n backup_file(self.conffile, backups=5, min_age_delta=900)\n if mailpile.platforms.RenameCannotOverwrite():\n try:\n # We only do this if we have to; we would rather just\n # use rename() as it's (more) atomic.\n os.remove(self.conffile)\n except (OSError, IOError):\n pass\n os.rename(newfile, self.conffile)\n\n # If we are shutting down, just stop here.\n if mailpile.util.QUITTING:\n return\n\n # Enable translations\n mailpile.i18n.ActivateTranslation(None, self, self.prefs.language)\n\n # Recreate VFS root in case new things have been configured\n self.vfs_root.rescan()\n\n # Reconfigure the connection broker\n from mailpile.conn_brokers import Master as ConnBroker\n ConnBroker.configure()\n\n # Notify workers that things have changed. We do this before\n # the prepare_workers() below, because we only want to notify\n # workers that were already running.\n self._unlocked_notify_workers_config_changed()\n\n # Prepare any new workers\n self.prepare_workers(daemons=self.daemons_started(), changed=True)\n\n # Invalidate command cache contents that depend on the config\n self.command_cache.mark_dirty([u'!config'])\n\n def _find_mail_source(self, mbx_id, path=None):\n if path:\n path = FilePath(path).raw_fp\n if path[:5] == '/src:':\n return self.sources[path[5:].split('/')[0]]\n if path[:4] == 'src:':\n return self.sources[path[4:].split('/')[0]]\n for src in self.sources.values():\n # Note: we cannot test 'mbx_id in ...' because of case sensitivity.\n if src.mailbox[FormatMbxId(mbx_id)] is not None:\n return src\n return None\n\n def get_mailboxes(self, with_mail_source=None,\n mail_source_locals=False):\n try:\n mailboxes = [(FormatMbxId(k),\n self.sys.mailbox[k],\n self._find_mail_source(k))\n for k in self.sys.mailbox.keys()\n if self.sys.mailbox[k] != '/dev/null']\n except (AttributeError):\n # Config not loaded, nothing to see here\n return []\n\n if with_mail_source is True:\n mailboxes = [(i, p, s) for i, p, s in mailboxes if s]\n elif with_mail_source is False:\n if mail_source_locals:\n mailboxes = [(i, p, s) for i, p, s in mailboxes\n if (not s) or (not s.enabled)]\n else:\n mailboxes = [(i, p, s) for i, p, s in mailboxes if not s]\n else:\n pass # All mailboxes, with or without mail sources\n\n if mail_source_locals:\n for i in range(0, len(mailboxes)):\n mid, path, src = mailboxes[i]\n mailboxes[i] = (mid,\n src and src.mailbox[mid].local or path,\n src)\n\n mailboxes.sort()\n return mailboxes\n\n def is_editable_message(self, msg_info):\n for ptr in msg_info[MailIndex.MSG_PTRS].split(','):\n if not self.is_editable_mailbox(ptr[:MBX_ID_LEN]):\n return False\n editable = False\n for tid in msg_info[MailIndex.MSG_TAGS].split(','):\n try:\n if self.tags and self.tags[tid].flag_editable:\n editable = True\n except (KeyError, AttributeError):\n pass\n return editable\n\n def is_editable_mailbox(self, mailbox_id):\n try:\n mailbox_id = ((mailbox_id is None and -1) or\n (mailbox_id == '' and -1) or\n int(mailbox_id, 36))\n local_mailbox_id = int(self.sys.get('local_mailbox_id', 'ZZZZZ'),\n 36)\n return (mailbox_id == local_mailbox_id)\n except ValueError:\n return False\n\n def load_pickle(self, pfn, delete_if_corrupt=False):\n pickle_path = os.path.join(self.workdir, pfn)\n try:\n with open(pickle_path, 'rb') as fd:\n master_key = self.get_master_key()\n if master_key:\n from mailpile.crypto.streamer import DecryptingStreamer\n with DecryptingStreamer(fd,\n mep_key=master_key,\n name='load_pickle(%s)' % pfn\n ) as streamer:\n data = streamer.read()\n streamer.verify(_raise=IOError)\n else:\n data = fd.read()\n return cPickle.loads(data)\n except (cPickle.UnpicklingError, IOError, EOFError, OSError):\n if delete_if_corrupt:\n safe_remove(pickle_path)\n raise IOError('Load/unpickle failed: %s' % pickle_path)\n\n def save_pickle(self, obj, pfn, encrypt=True):\n ppath = os.path.join(self.workdir, pfn)\n if encrypt and self.get_master_key() and self.prefs.encrypt_misc:\n from mailpile.crypto.streamer import EncryptingStreamer\n with EncryptingStreamer(self.get_master_key(),\n dir=self.tempfile_dir(),\n header_data={'subject': pfn},\n name='save_pickle') as fd:\n cPickle.dump(obj, fd, protocol=0)\n fd.save(ppath)\n else:\n with open(ppath, 'wb') as fd:\n cPickle.dump(obj, fd, protocol=0)\n\n def _mailbox_info(self, mailbox_id, prefer_local=True):\n try:\n with self._lock:\n mbx_id = FormatMbxId(mailbox_id)\n mfn = self.sys.mailbox[mbx_id]\n src = self._find_mail_source(mailbox_id, path=mfn)\n pfn = 'pickled-mailbox.%s' % mbx_id.lower()\n if prefer_local and src and src.mailbox[mbx_id] is not None:\n mfn = src and src.mailbox[mbx_id].local or mfn\n else:\n pfn += '-R'\n except (KeyError, TypeError):\n traceback.print_exc()\n raise NoSuchMailboxError(_('No such mailbox: %s') % mailbox_id)\n return mbx_id, src, FilePath(mfn), pfn\n\n def save_mailbox(self, session, pfn, mbox):\n if pfn is not None:\n mbox.save(session,\n to=pfn, pickler=lambda o, f: self.save_pickle(o, f))\n\n def uncache_mailbox(self, session, entry, drop=True, force_save=False):\n \"\"\"\n Safely remove a mailbox from the cache, saving any state changes to\n the encrypted pickles.\n\n If the mailbox is still in use somewhere in the app (as measured by\n the Python reference counter), we DON'T remove from cache, to ensure\n each mailbox is represented by exactly one object at a time.\n \"\"\"\n pfn, mbx_id = entry[:2] # Don't grab mbox, to not add more refs\n if pfn:\n def dropit(l):\n return [c for c in l if (c[0] != pfn)]\n else:\n def dropit(l):\n return [c for c in l if (c[1] != mbx_id)]\n\n with self._lock:\n mboxes = [c[2] for c in self._mbox_cache\n if ((c[0] == pfn) if pfn else (c[1] == mbx_id))]\n if len(mboxes) < 1:\n # Not found, nothing to do here\n return\n\n # At this point, if the mailbox is not in use, there should be\n # exactly 2 references to it: in mboxes and self._mbox_cache.\n # However, sys.getrefcount always returns one extra for itself.\n if sys.getrefcount(mboxes[0]) > 3:\n if force_save:\n self.save_mailbox(session, pfn, mboxes[0])\n return\n\n # This may be slow, but it has to happen inside the lock\n # otherwise we run the risk of races.\n self.save_mailbox(session, pfn, mboxes[0])\n\n if drop:\n self._mbox_cache = dropit(self._mbox_cache)\n else:\n keep2 = self._mbox_cache[-MAX_CACHED_MBOXES:]\n keep1 = dropit(self._mbox_cache[:-MAX_CACHED_MBOXES])\n self._mbox_cache = keep1 + keep2\n\n def cache_mailbox(self, session, pfn, mbx_id, mbox):\n \"\"\"\n Add a mailbox to the cache, potentially evicting other entries if the\n cache has grown too large.\n \"\"\"\n with self._lock:\n if pfn is not None:\n self._mbox_cache = [\n c for c in self._mbox_cache if c[0] != pfn]\n elif mbx_id:\n self._mbox_cache = [\n c for c in self._mbox_cache if c[1] != mbx_id]\n self._mbox_cache.append((pfn, mbx_id, mbox))\n flush = self._mbox_cache[:-MAX_CACHED_MBOXES]\n for entry in flush:\n pfn, mbx_id = entry[:2]\n self.save_worker.add_unique_task(\n session, 'Save mailbox %s/%s (drop=%s)' % (mbx_id, pfn, False),\n lambda: self.uncache_mailbox(session, entry, drop=False))\n\n def flush_mbox_cache(self, session, clear=True, wait=False):\n if wait:\n saver = self.save_worker.do\n else:\n saver = self.save_worker.add_task\n with self._lock:\n flush = self._mbox_cache[:]\n for entry in flush:\n pfn, mbx_id = entry[:2]\n saver(session,\n 'Save mailbox %s/%s (drop=%s)' % (mbx_id, pfn, clear),\n lambda: self.uncache_mailbox(session, entry,\n drop=clear, force_save=True),\n unique=True)\n\n def find_mboxids_and_sources_by_path(self, *paths):\n def _au(p):\n return unicode(p[1:] if (p[:5] == '/src:') else\n p if (p[:4] == 'src:') else\n vfs.abspath(p))\n abs_paths = dict((_au(p), [p]) for p in paths)\n with self._lock:\n for sid, src in self.sources.iteritems():\n for mid, info in src.mailbox.iteritems():\n umfn = _au(self.sys.mailbox[mid])\n if umfn in abs_paths:\n abs_paths[umfn].append((mid, src))\n if info.local:\n lmfn = _au(info.local)\n if lmfn in abs_paths:\n abs_paths[lmfn].append((mid, src))\n\n for mid, mfn in self.sys.mailbox.iteritems():\n umfn = _au(mfn)\n if umfn in abs_paths:\n if umfn[:4] == u'src:':\n src = self.sources.get(umfn[4:].split('/')[0])\n else:\n src = None\n abs_paths[umfn].append((mid, src))\n\n return dict((p[0], p[1]) for p in abs_paths.values() if p[1:])\n\n def open_mailbox_path(self, session, path, register=False, raw_open=False):\n path = vfs.abspath(path)\n mbox = mbx_mid = mbx_src = None\n with self._lock:\n msmap = self.find_mboxids_and_sources_by_path(unicode(path))\n if msmap:\n mbx_mid, mbx_src = list(msmap.values())[0]\n\n if (register or raw_open) and mbx_mid is None:\n mbox = dict(((i, m) for p, i, m in self._mbox_cache)\n ).get(path, None)\n\n if path.raw_fp.startswith('/src:'):\n path = FilePath(path.raw_fp[1:])\n\n if mbox:\n pass\n elif path.raw_fp.startswith('src:'):\n msrc_id = path.raw_fp[4:].split('/')[0]\n msrc = self.mail_sources.get(msrc_id)\n if msrc:\n mbox = msrc.open_mailbox(None, path.raw_fp)\n else:\n mbox = OpenMailbox(path.raw_fp, self, create=False)\n\n if register:\n mbx_mid = self.sys.mailbox.append(unicode(path))\n mbox = None # Force a re-open below\n\n elif mbox:\n # (re)-add to the cache; we need to do this here\n # because we did the opening ourselves instead of\n # invoking open_mailbox as below.\n self.cache_mailbox(session, None, path.raw_fp, mbox)\n\n if mbx_mid is not None:\n mbx_mid = FormatMbxId(mbx_mid)\n if mbox is None:\n mbox = self.open_mailbox(session, mbx_mid, prefer_local=True)\n return (mbx_mid, mbox)\n\n elif raw_open and mbox:\n return (mbx_mid, mbox)\n\n raise ValueError('Not found')\n\n def open_mailbox(self, session, mailbox_id,\n prefer_local=True, from_cache=False):\n mbx_id, src, mfn, pfn = self._mailbox_info(mailbox_id,\n prefer_local=prefer_local)\n with self._lock:\n mbox = dict(((p, m) for p, i, m in self._mbox_cache)\n ).get(pfn, None)\n try:\n if mbox is None:\n if from_cache:\n return None\n if session:\n session.ui.mark(_('%s: Updating: %s') % (mbx_id, mfn))\n mbox = self.load_pickle(pfn, delete_if_corrupt=True)\n if prefer_local and not mbox.is_local:\n mbox = None\n else:\n mbox.update_toc()\n except AttributeError:\n mbox = None\n except KeyboardInterrupt:\n raise\n except IOError:\n mbox = None\n except:\n if self.sys.debug:\n traceback.print_exc()\n mbox = None\n\n if mbox is None:\n if session:\n session.ui.mark(_('%s: Opening: %s (may take a while)'\n ) % (mbx_id, mfn))\n editable = self.is_editable_mailbox(mbx_id)\n if src is not None:\n msrc = self.mail_sources.get(src._key)\n mbox = msrc.open_mailbox(mbx_id, mfn.raw_fp) if msrc else None\n if mbox is None:\n mbox = OpenMailbox(mfn.raw_fp, self, create=editable)\n mbox.editable = editable\n mbox.is_local = prefer_local\n\n # Always set these, they can't be pickled\n mbox._decryption_key_func = lambda: self.get_master_key()\n mbox._encryption_key_func = lambda: (self.get_master_key() if\n self.prefs.encrypt_mail else None)\n\n # Finally, re-add to the cache\n self.cache_mailbox(session, pfn, mbx_id, mbox)\n\n return mbox\n\n def create_local_mailstore(self, session, name=None):\n path = os.path.join(self.workdir, 'mail')\n with self._lock:\n if name is None:\n name = '%5.5x' % random.randint(0, 16**5)\n while os.path.exists(os.path.join(path, name)):\n name = '%5.5x' % random.randint(0, 16**5)\n if name != '':\n if not os.path.exists(path):\n root_mbx = wervd.MailpileMailbox(path)\n if name.startswith(path) and '..' not in name:\n path = name\n else:\n path = os.path.join(path, os.path.basename(name))\n\n mbx = wervd.MailpileMailbox(path)\n mbx._decryption_key_func = lambda: self.get_master_key()\n mbx._encryption_key_func = lambda: (self.get_master_key() if\n self.prefs.encrypt_mail else None)\n return FilePath(path), mbx\n\n def open_local_mailbox(self, session):\n with self._lock:\n local_id = self.sys.get('local_mailbox_id', None)\n if local_id is None or local_id == '':\n mailbox, mbx = self.create_local_mailstore(session, name='')\n local_id = FormatMbxId(self.sys.mailbox.append(mailbox))\n self.sys.local_mailbox_id = local_id\n else:\n local_id = FormatMbxId(local_id)\n return local_id, self.open_mailbox(session, local_id)\n\n def get_passphrase(self, keyid,\n description=None, prompt=None, error=None,\n no_ask=False, no_cache=False):\n if not no_cache:\n keyidL = keyid.lower()\n for sid in self.secrets.keys():\n if sid.endswith(keyidL):\n secret = self.secrets[sid]\n if secret.policy == 'always-ask':\n no_cache = True\n elif secret.policy == 'fail':\n return False, None\n elif secret.policy != 'cache-only':\n sps = SecurePassphraseStorage(secret.password)\n return (keyidL, sps)\n\n if not no_cache:\n if keyid in self.passphrases:\n return (keyid, self.passphrases[keyid])\n if keyidL in self.passphrases:\n return (keyidL, self.passphrases[keyidL])\n for fprint in self.passphrases:\n if fprint.endswith(keyid):\n return (fprint, self.passphrases[fprint])\n if fprint.lower().endswith(keyidL):\n return (fprint, self.passphrases[fprint])\n\n if not no_ask:\n # This will either record details to the event of the currently\n # running command/operation, or register a new event. This does\n # not work as one might hope if ops cross a thread boundary...\n ev = GetThreadEvent(\n create=True,\n message=prompt or _('Please enter your password'),\n source=self)\n\n details = {'id': keyid}\n if prompt: details['msg'] = prompt\n if error: details['err'] = error\n if description: details['dsc'] = description\n if 'password_needed' in ev.private_data:\n ev.private_data['password_needed'].append(details)\n else:\n ev.private_data['password_needed'] = [details]\n\n ev.data['password_needed'] = True\n\n # Post a password request to the event log...\n self.event_log.log_event(ev)\n\n return None, None\n\n def get_profile(self, email=None):\n find = email or self.prefs.get('default_email', None)\n default_sig = _('Sent using Mailpile, Free Software '\n 'from www.mailpile.is')\n default_profile = {\n 'name': None,\n 'email': find,\n 'messageroute': self.prefs.default_messageroute,\n 'signature': default_sig,\n 'crypto_policy': 'none',\n 'crypto_format': 'none',\n 'vcard': None\n }\n\n profiles = []\n if find:\n profiles = [self.vcards.get_vcard(find)]\n if not profiles or not profiles[0]:\n profiles = self.vcards.find_vcards([], kinds=['profile'])\n if profiles:\n profiles.sort(key=lambda k: ((0 if k.route else 1),\n (-len(k.recent_history())),\n (-len(k.sources()))))\n\n if profiles and profiles[0]:\n profile = profiles[0]\n psig = profile.signature\n proute = profile.route\n default_profile.update({\n 'name': profile.fn,\n 'email': find or profile.email,\n 'signature': psig if (psig is not None) else default_sig,\n 'messageroute': (proute if (proute is not None)\n else self.prefs.default_messageroute),\n 'crypto_policy': profile.crypto_policy or 'none',\n 'crypto_format': profile.crypto_format or 'none',\n 'vcard': profile\n })\n\n return default_profile\n\n def get_route(self, frm, rcpts=['-t']):\n if len(rcpts) == 1:\n if rcpts[0].lower().endswith('.onion'):\n return {\"protocol\": \"smtorp\",\n \"host\": rcpts[0].split('@')[-1],\n \"port\": 25,\n \"auth_type\": \"\",\n \"username\": \"\",\n \"password\": \"\"}\n routeid = self.get_profile(frm)['messageroute']\n if self.routes[routeid] is not None:\n return self.routes[routeid]\n else:\n raise ValueError(_(\"Route %s for %s does not exist.\"\n ) % (routeid, frm))\n\n def data_directory(self, ftype, mode='rb', mkdir=False):\n \"\"\"\n Return the path to a data directory for a particular type of file\n data, optionally creating the directory if it is missing.\n\n >>> p = cfg.data_directory('html_theme', mode='r', mkdir=False)\n >>> p == os.path.abspath('shared-data/default-theme')\n True\n \"\"\"\n # This should raise a KeyError if the ftype is unrecognized\n bpath = self.sys.path.get(ftype)\n if not bpath.startswith('/'):\n cpath = os.path.join(self.workdir, bpath)\n if os.path.exists(cpath) or 'w' in mode:\n bpath = cpath\n if mkdir and not os.path.exists(cpath):\n os.mkdir(cpath)\n else:\n bpath = os.path.join(self.shareddatadir, bpath)\n return os.path.abspath(bpath)\n\n def data_file_and_mimetype(self, ftype, fpath, *args, **kwargs):\n # The theme gets precedence\n core_path = self.data_directory(ftype, *args, **kwargs)\n path, mimetype = os.path.join(core_path, fpath), None\n\n # If there's still nothing there, check our plugins\n if not os.path.exists(path):\n from mailpile.plugins import PluginManager\n path, mimetype = PluginManager().get_web_asset(fpath, path)\n\n if os.path.exists(path):\n return path, mimetype\n else:\n return None, None\n\n def history_file(self):\n return os.path.join(self.workdir, 'history')\n\n def mailindex_file(self):\n return os.path.join(self.workdir, 'mailpile.idx')\n\n def mailpile_path(self, path):\n base = (self.workdir + os.sep).replace(os.sep+os.sep, os.sep)\n if path.startswith(base):\n return path[len(base):]\n\n rbase = os.path.realpath(base) + os.sep\n rpath = os.path.realpath(path)\n if rpath.startswith(rbase):\n return rpath[len(rbase):]\n\n return path\n\n def tempfile_dir(self):\n d = os.path.join(self.workdir, 'tmp')\n if not os.path.exists(d):\n os.mkdir(d)\n return d\n\n def clean_tempfile_dir(self):\n try:\n td = self.tempfile_dir()\n files = os.listdir(td)\n random.shuffle(files)\n for fn in files:\n fn = os.path.join(td, fn)\n if os.path.isfile(fn):\n safe_remove(fn)\n except (OSError, IOError):\n pass\n\n def postinglist_dir(self, prefix):\n d = os.path.join(self.workdir, 'search')\n if not os.path.exists(d):\n os.mkdir(d)\n d = os.path.join(d, prefix and prefix[0] or '_')\n if not os.path.exists(d):\n os.mkdir(d)\n return d\n\n def need_more_disk_space(self, required=0, nodefault=False, ratio=1.0):\n \"\"\"Returns a path where we need more disk space, None if all is ok.\"\"\"\n if self.detected_memory_corruption:\n return '/'\n if not (nodefault and required):\n required = ratio * max(required, self.sys.minfree_mb * 1024 * 1024)\n for path in (self.workdir, ):\n if get_free_disk_bytes(path) < required:\n return path\n return None\n\n def interruptable_wait_for_lock(self):\n # This construct allows the user to CTRL-C out of things.\n delay = 0.01\n while self._lock.acquire(False) == False:\n if mailpile.util.QUITTING:\n raise KeyboardInterrupt('Quitting')\n time.sleep(delay)\n delay = min(1, delay*2)\n self._lock.release()\n return self._lock\n\n def get_index(self, session):\n # Note: This is a long-running lock, but having two sets of the\n # index would really suck and this should only ever happen once.\n with self.interruptable_wait_for_lock():\n if self.index:\n return self.index\n self.index_loading = MailIndex(self)\n self.index_loading.load(session)\n self.index = self.index_loading\n self.index_loading = None\n try:\n self.index_check.release()\n except:\n pass\n return self.index\n\n def get_path_index(self, session, path):\n \"\"\"\n Get a search index by path (instead of the default), or None if\n no matching index is found.\n \"\"\"\n idx = None\n\n mi, mbox = self.open_mailbox_path(session, path, raw_open=True)\n if mbox:\n idx = mbox.get_index(self, mbx_mid=mi)\n\n # Return a sad, boring, empty index.\n if idx is None:\n import mailpile.index.base\n idx = mailpile.index.base.BaseIndex(self)\n\n return idx\n\n def get_proxy_settings(self):\n if self.sys.proxy.protocol == 'system':\n proxy_list = getproxies()\n for proto in ('socks5', 'socks4', 'http'):\n for url in proxy_list.values():\n if url.lower().startswith(proto+'://'):\n try:\n p, host, port = url.replace('/', '').split(':')\n return {\n 'protocol': proto,\n 'fallback': self.sys.proxy.fallback,\n 'host': host,\n 'port': int(port),\n 'no_proxy': self.sys.proxy.no_proxy}\n except (ValueError, IndexError, KeyError):\n pass\n elif self.sys.proxy.protocol in ('tor', 'tor-risky'):\n if self.tor_worker is not None:\n return {\n 'protocol': self.sys.proxy.protocol,\n 'fallback': self.sys.proxy.fallback,\n 'host': '127.0.0.1',\n 'port': self.tor_worker.socks_port,\n 'no_proxy': self.sys.proxy.no_proxy}\n return self.sys.proxy\n\n def open_file(self, ftype, fpath, mode='rb', mkdir=False):\n if '..' in fpath:\n raise ValueError(_('Parent paths are not allowed'))\n fpath, mt = self.data_file_and_mimetype(ftype, fpath,\n mode=mode, mkdir=mkdir)\n if not fpath:\n raise IOError(2, 'Not Found')\n return fpath, open(fpath, mode), mt\n\n def daemons_started(config, which=None):\n return ((which or config.save_worker)\n not in (None, config.dumb_worker))\n\n def get_mail_source(config, src_id, start=False, changed=False):\n ms_thread = config.mail_sources.get(src_id)\n if (ms_thread and not ms_thread.isAlive()):\n ms_thread = None\n if not ms_thread:\n from mailpile.mail_source import MailSource\n src_config = config.sources[src_id]\n ms_thread = MailSource(config.background, src_config)\n if start:\n config.mail_sources[src_id] = ms_thread\n ms_thread.start()\n if changed:\n ms_thread.wake_up()\n return ms_thread\n\n def start_tor_worker(config):\n from mailpile.conn_brokers import Master as ConnBroker\n from mailpile.crypto.tor import Tor\n config.tor_worker = Tor(\n config=config, session=config.background,\n callbacks=[lambda c: ConnBroker.configure()])\n config.tor_worker.start()\n return config.tor_worker\n\n def prepare_workers(self, *args, **kwargs):\n with self._lock:\n return self._unlocked_prepare_workers(*args, **kwargs)\n\n def _unlocked_prepare_workers(config, session=None, changed=False,\n daemons=False, httpd_spec=None):\n\n # Set our background UI to something that can log.\n if session:\n config.background.ui = BackgroundInteraction(\n config, log_parent=session.ui)\n\n # Tell conn broker that we exist\n from mailpile.conn_brokers import Master as ConnBroker\n ConnBroker.set_config(config)\n if 'connbroker' in config.sys.debug:\n ConnBroker.debug_callback = lambda msg: config.background.ui.debug(msg)\n else:\n ConnBroker.debug_callback = None\n\n def start_httpd(sspec=None):\n sspec = sspec or (config.sys.http_host, config.sys.http_port,\n config.sys.http_path or '')\n if sspec[0].lower() != 'disabled' and sspec[1] >= 0:\n try:\n if mailpile.platforms.NeedExplicitPortCheck():\n try:\n socket.socket().connect((sspec[0],sspec[1]))\n port_in_use = True\n except socket.error:\n port_in_use = False\n if port_in_use:\n raise socket.error(errno.EADDRINUSE)\n config.http_worker = HttpWorker(config.background, sspec)\n config.http_worker.start()\n except socket.error as e:\n if e[0] == errno.EADDRINUSE:\n session.ui.error(\n _('Port %s:%s in use by another Mailpile or program'\n ) % (sspec[0], sspec[1]))\n\n # We may start the HTTPD without the loaded config...\n if not config.loaded_config:\n if daemons and not config.http_worker:\n start_httpd(httpd_spec)\n return\n\n # Start the other workers\n if daemons:\n for src_id in config.sources.keys():\n try:\n config.get_mail_source(src_id, start=True, changed=changed)\n except (ValueError, KeyError):\n pass\n\n should_launch_tor = ((not config.sys.tor.systemwide)\n and (config.sys.proxy.protocol.startswith('tor')))\n if config.tor_worker is None:\n if should_launch_tor:\n config.start_tor_worker()\n elif not should_launch_tor:\n config.tor_worker.stop_tor()\n config.tor_worker = None\n\n if config.slow_worker == config.dumb_worker:\n config.slow_worker = Worker('Slow worker', config.background)\n config.slow_worker.wait_until = lambda: (\n (not config.save_worker) or config.save_worker.is_idle())\n config.slow_worker.start()\n if config.scan_worker == config.dumb_worker:\n config.scan_worker = Worker('Scan worker', config.background)\n config.slow_worker.wait_until = lambda: (\n (not config.save_worker) or config.save_worker.is_idle())\n config.scan_worker.start()\n if config.save_worker == config.dumb_worker:\n config.save_worker = ImportantWorker('Save worker',\n config.background)\n config.save_worker.start()\n if not config.cron_worker:\n config.cron_worker = Cron(\n config.cron_schedule, 'Cron worker', config.background)\n config.cron_worker.start()\n if not config.http_worker:\n start_httpd(httpd_spec)\n if not config.other_workers:\n from mailpile.plugins import PluginManager\n for worker in PluginManager.WORKERS:\n w = worker(config.background)\n w.start()\n config.other_workers.append(w)\n\n # Update the cron jobs, if necessary\n if config.cron_worker and config.event_log:\n from mailpile.postinglist import GlobalPostingList\n from mailpile.plugins.core import HealthCheck\n def gpl_optimize():\n if HealthCheck.check(config.background, config):\n rs_interval = (config.prefs.rescan_interval or 1800)\n runtime = rs_interval / 10\n ratio = 2.0 / (7*24*3600.0 / rs_interval) # Optimize 2x/week\n config.slow_worker.add_unique_task(\n config.background, 'Optimize GPL',\n lambda: GlobalPostingList.Optimize(\n config.background,\n config.index,\n lazy=(not user_probably_asleep()),\n ratio=ratio,\n runtime=runtime))\n\n # Schedule periodic rescanning, if requested.\n rescan_interval = config.prefs.rescan_interval\n if rescan_interval:\n def rescan():\n from mailpile.plugins.core import Rescan\n if 'rescan' not in config._running:\n rsc = Rescan(config.background, 'rescan')\n rsc.serialize = False\n config.slow_worker.add_unique_task(\n config.background, 'Rescan',\n lambda: rsc.run(slowly=True, cron=True))\n gpl_optimize()\n config.cron_worker.add_task('rescan', rescan_interval, rescan)\n else:\n config.cron_worker.add_task('gpl_optimize', 1800, gpl_optimize)\n\n def metadata_index_saver():\n config.save_worker.add_unique_task(\n config.background, 'save_metadata_index',\n lambda: config.index.save_changes())\n config.cron_worker.add_task(\n 'save_metadata_index', 900, metadata_index_saver)\n\n def search_history_saver():\n config.save_worker.add_unique_task(\n config.background, 'save_search_history',\n lambda: config.search_history.save(config))\n config.cron_worker.add_task(\n 'save_search_history', 900, search_history_saver)\n\n def refresh_command_cache():\n config.scan_worker.add_unique_task(\n config.background, 'refresh_command_cache',\n lambda: config.command_cache.refresh(\n event_log=config.event_log),\n first=True)\n config.cron_worker.add_task(\n 'refresh_command_cache', 5, refresh_command_cache)\n\n # Schedule plugin jobs\n from mailpile.plugins import PluginManager\n\n def interval(i):\n if isinstance(i, (str, unicode)):\n i = config.walk(i)\n return int(i)\n\n def wrap_fast(func):\n def wrapped():\n return func(config.background)\n return wrapped\n\n def wrap_slow(func):\n def wrapped():\n config.slow_worker.add_unique_task(\n config.background, job,\n lambda: func(config.background))\n return wrapped\n for job, (i, f) in PluginManager.FAST_PERIODIC_JOBS.iteritems():\n config.cron_worker.add_task(job, interval(i), wrap_fast(f))\n for job, (i, f) in PluginManager.SLOW_PERIODIC_JOBS.iteritems():\n config.cron_worker.add_task(job, interval(i), wrap_slow(f))\n\n def _unlocked_get_all_workers(config):\n return (config.mail_sources.values() +\n config.other_workers +\n [config.http_worker,\n config.tor_worker,\n config.slow_worker,\n config.scan_worker,\n config.cron_worker])\n\n def stop_workers(config):\n try:\n self.index_check.release()\n except:\n pass\n\n with config._lock:\n worker_list = config._unlocked_get_all_workers()\n config.other_workers = []\n config.tor_worker = None\n config.http_worker = None\n config.cron_worker = None\n config.slow_worker = config.dumb_worker\n config.scan_worker = config.dumb_worker\n\n for wait in (False, True):\n for w in worker_list:\n if w and w.isAlive():\n if config.sys.debug and wait:\n print('Waiting for %s' % w)\n w.quit(join=wait)\n\n # Flush the mailbox cache (queues save worker jobs)\n config.flush_mbox_cache(config.background, clear=True)\n\n # Handle the save worker last, once all the others are\n # no longer feeding it new things to do.\n with config._lock:\n save_worker = config.save_worker\n config.save_worker = config.dumb_worker\n if config.sys.debug:\n print('Waiting for %s' % save_worker)\n\n from mailpile.postinglist import PLC_CACHE_FlushAndClean\n PLC_CACHE_FlushAndClean(config.background, keep=0)\n config.search_history.save(config)\n save_worker.quit(join=True)\n\n if config.sys.debug:\n # Hooray!\n print('All stopped!')\n\n def _unlocked_notify_workers_config_changed(config):\n worker_list = config._unlocked_get_all_workers()\n for worker in worker_list:\n if hasattr(worker, 'notify_config_changed'):\n worker.notify_config_changed()\n\n\n##############################################################################\n\nif __name__ == \"__main__\":\n import copy\n import doctest\n import sys\n import mailpile.config.base\n import mailpile.config.defaults\n import mailpile.config.manager\n import mailpile.plugins.tags\n import mailpile.ui\n\n rules = copy.deepcopy(mailpile.config.defaults.CONFIG_RULES)\n rules.update({\n 'nest1': ['Nest1', {\n 'nest2': ['Nest2', str, []],\n 'nest3': ['Nest3', {\n 'nest4': ['Nest4', str, []]\n }, []],\n }, {}]\n })\n cfg = mailpile.config.manager.ConfigManager(rules=rules)\n session = mailpile.ui.Session(cfg)\n session.ui = mailpile.ui.SilentInteraction(cfg)\n session.ui.block()\n\n for tries in (1, 2):\n # This tests that we can set (and reset) dicts of unnested objects\n cfg.tags = {}\n assert(cfg.tags.a is None)\n for tn in range(0, 11):\n cfg.tags.append({'name': 'Test Tag %s' % tn})\n assert(cfg.tags.a['name'] == 'Test Tag 10')\n\n # This tests the same thing for lists\n #cfg.profiles = []\n #assert(len(cfg.profiles) == 0)\n #cfg.profiles.append({'name': 'Test Profile'})\n #assert(len(cfg.profiles) == 1)\n #assert(cfg.profiles[0].name == 'Test Profile')\n\n # This is the complicated one: multiple nesting layers\n cfg.nest1 = {}\n assert(cfg.nest1.a is None)\n cfg.nest1.a = {\n 'nest2': ['hello', 'world'],\n 'nest3': [{'nest4': ['Hooray']}]\n }\n cfg.nest1.b = {\n 'nest2': ['hello', 'world'],\n 'nest3': [{'nest4': ['Hooray', 'Bravo']}]\n }\n assert(cfg.nest1.a.nest3[0].nest4[0] == 'Hooray')\n assert(cfg.nest1.b.nest3[0].nest4[1] == 'Bravo')\n\n assert(cfg.sys.http_port ==\n mailpile.config.defaults.CONFIG_RULES['sys'][-1]['http_port'][-1])\n assert(cfg.sys.path.vcards == 'vcards')\n assert(cfg.walk('sys.path.vcards') == 'vcards')\n\n # Verify that the tricky nested stuff from above persists and\n # load/save doesn't change lists.\n for passes in (1, 2, 3):\n cfg2 = mailpile.config.manager.ConfigManager(rules=rules)\n cfg2.parse_config(session, cfg.as_config_bytes())\n cfg.parse_config(session, cfg2.as_config_bytes())\n assert(cfg2.nest1.a.nest3[0].nest4[0] == 'Hooray')\n assert(cfg2.nest1.b.nest3[0].nest4[1] == 'Bravo')\n assert(len(cfg2.nest1) == 2)\n assert(len(cfg.nest1) == 2)\n assert(len(cfg.tags) == 11)\n\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={'cfg': cfg,\n 'session': session})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/config/paths.py", "content": "import os\nimport sys\n\nimport mailpile.platforms\n\ntry:\n from appdirs import AppDirs\nexcept ImportError:\n AppDirs = None\n\n\ndef _ensure_exists(path, mode=0o700):\n if not os.path.exists(path):\n head, tail = os.path.split(path)\n _ensure_exists(head)\n os.mkdir(path, mode)\n return path\n\n\ndef LEGACY_DEFAULT_WORKDIR(profile):\n if profile == 'default':\n # Backwards compatibility: If the old ~/.mailpile exists, use it.\n workdir = os.path.expanduser('~/.mailpile')\n if os.path.exists(workdir) and os.path.isdir(workdir):\n return workdir\n\n return os.path.join(\n mailpile.platforms.GetAppDataDirectory(), 'Mailpile', profile)\n\n\ndef DEFAULT_WORKDIR():\n # The Mailpile environment variable trumps everything\n workdir = os.getenv('MAILPILE_HOME')\n if workdir:\n return _ensure_exists(workdir)\n\n # Which profile?\n profile = os.getenv('MAILPILE_PROFILE', 'default')\n\n # Check if we have a legacy setup we need to preserve\n workdir = LEGACY_DEFAULT_WORKDIR(profile)\n if not AppDirs or (os.path.exists(workdir) and os.path.isdir(workdir)):\n return _ensure_exists(workdir)\n\n # Use platform-specific defaults\n # via https://github.com/ActiveState/appdirs\n dirs = AppDirs(\"Mailpile\", \"Mailpile ehf\")\n return _ensure_exists(os.path.join(dirs.user_data_dir, profile))\n\n\ndef DEFAULT_SHARED_DATADIR():\n # IMPORTANT: This code is duplicated in mailpile-admin.py.\n # If it needs changing please change both places!\n env_share = os.getenv('MAILPILE_SHARED')\n if env_share is not None:\n return env_share\n\n # Check if we are running in a virtual env\n # http://stackoverflow.com/questions/1871549/python-determine-if-running-inside-virtualenv\n # We must also check that we are installed in the virtual env,\n # not just that we are running in a virtual env.\n if ((hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix'))\n and __file__.startswith(sys.prefix)):\n return os.path.join(sys.prefix, 'share', 'mailpile')\n\n # Check if we've been installed to /usr/local (or equivalent)\n usr_local = os.path.join(sys.prefix, 'local')\n if __file__.startswith(usr_local):\n return os.path.join(usr_local, 'share', 'mailpile')\n\n # Check if we are in /usr/ (sys.prefix)\n if __file__.startswith(sys.prefix):\n return os.path.join(sys.prefix, 'share', 'mailpile')\n\n # Else assume dev mode, source tree layout\n return os.path.join(\n os.path.dirname(__file__), '..', '..', 'shared-data')\n\n\ndef DEFAULT_LOCALE_DIRECTORY():\n \"\"\"Get the gettext translation object, no matter where our CWD is\"\"\"\n return os.path.join(DEFAULT_SHARED_DATADIR(), \"locale\")\n\n\ndef LOCK_PATHS(workdir=None):\n if workdir is None:\n workdir = DEFAULT_WORKDIR()\n return (\n os.path.join(workdir, 'public-lock'),\n os.path.join(workdir, 'workdir-lock'))\n" }, { "path": "mailpile/config/validators.py", "content": "from __future__ import print_function\nimport os\nimport socket\nimport re\n\ntry:\n import win_inet_pton\nexcept ImportError:\n pass\n\nfrom urlparse import urlparse\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as n\nfrom mailpile.util import *\n\n\ndef BoolCheck(value):\n \"\"\"\n Convert common yes/no strings into boolean values.\n\n >>> BoolCheck('yes')\n True\n >>> BoolCheck('no')\n False\n\n >>> BoolCheck('true')\n True\n >>> BoolCheck('false')\n False\n\n >>> BoolCheck('on')\n True\n >>> BoolCheck('off')\n False\n\n >>> BoolCheck('wiggle')\n Traceback (most recent call last):\n ...\n ValueError: Invalid boolean: wiggle\n \"\"\"\n bool_val = truthy(value, default=None)\n if bool_val is None:\n raise ValueError(_('Invalid boolean: %s') % value)\n return bool_val\n\n\ndef SlugCheck(slug, allow=''):\n \"\"\"\n Verify that a string is a valid URL slug.\n\n >>> SlugCheck('_Foo-bar.5')\n '_foo-bar.5'\n\n >>> SlugCheck('Bad Slug')\n Traceback (most recent call last):\n ...\n ValueError: Invalid URL slug: Bad Slug\n\n >>> SlugCheck('Bad/Slug')\n Traceback (most recent call last):\n ...\n ValueError: Invalid URL slug: Bad/Slug\n \"\"\"\n if not slug == CleanText(unicode(slug),\n banned=(CleanText.NONDNS.replace(allow, ''))\n ).clean:\n raise ValueError(_('Invalid URL slug: %s') % slug)\n return slug.lower()\n\n\ndef SlashSlugCheck(slug):\n \"\"\"\n Verify that a string is a valid URL slug (slashes allowed).\n\n >>> SlashSlugCheck('Okay/Slug')\n 'okay/slug'\n \"\"\"\n return SlugCheck(slug, allow='/')\n\n\ndef RouteProtocolCheck(proto):\n \"\"\"\n Verify that the protocol is actually a protocol.\n (FIXME: Should reference a list of registered protocols...)\n\n >>> RouteProtocolCheck('SMTP')\n 'smtp'\n \"\"\"\n proto = str(proto).strip().lower()\n if proto not in (\"smtp\", \"smtptls\", \"smtpssl\", \"local\"):\n raise ValueError(_('Invalid message delivery protocol: %s') % proto)\n return proto\n\ndef DnsNameValid(dnsname):\n \"\"\"\n Tests whether a string is a valid dns name, returns a boolean value\n \"\"\"\n if not dnsname or not DNSNAME_RE.match(dnsname):\n return False\n else:\n return True\n\ndef HostNameValid(host):\n \"\"\"\n Tests whether a string is a valid host-name, return a boolean value\n\n >>> HostNameValid(\"127.0.0.1\")\n True\n\n >>> HostNameValid(\"::1\")\n True\n\n >>> HostNameValid(\"localhost\")\n True\n\n >>> HostNameValid(\"22.45\")\n False\n \"\"\"\n valid = False\n for attr in [\"AF_INET\",\"AF_INET6\"]:\n try:\n socket.inet_pton(socket.__getattribute__(attr), host)\n valid = True\n break\n except (socket.error):\n pass\n if not valid:\n # the host is not an IP so check if its a hostname i.e. 'localhost' or 'site.com'\n if not host or (not DnsNameValid(host) and not ALPHA_RE.match(host)):\n return False\n else:\n return True\n else:\n return True\n\ndef HostNameCheck(host):\n \"\"\"\n Verify that a string is a valid host-name, return it lowercased.\n\n >>> HostNameCheck('foo.BAR.baz')\n 'foo.bar.baz'\n\n >>> HostNameCheck('127.0.0.1')\n '127.0.0.1'\n\n >>> HostNameCheck('not/a/hostname')\n Traceback (most recent call last):\n ...\n ValueError: Invalid hostname: not/a/hostname\n \"\"\"\n # Check DNS, IPv4, and finally IPv6\n if not HostNameValid(host):\n raise ValueError(_('Invalid hostname: %s') % host)\n return str(host).lower()\n\n\ndef B36Check(b36val):\n \"\"\"\n Verify that a string is a valid path base-36 integer.\n\n >>> B36Check('Aa')\n 'aa'\n\n >>> B36Check('.')\n Traceback (most recent call last):\n ...\n ValueError: invalid ...\n \"\"\"\n int(b36val, 36)\n return str(b36val).lower()\n\n\ndef NotUnicode(string):\n \"\"\"\n Make sure a string is NOT unicode.\n \"\"\"\n if isinstance(string, unicode):\n string = string.encode('utf-8')\n if not isinstance(string, str):\n return str(string)\n return string\n\n\ndef PathCheck(path):\n \"\"\"\n Verify that a string is a valid path, make it absolute.\n\n >>> PathCheck('/etc/../')\n '/'\n\n >>> PathCheck('/no/such/path')\n Traceback (most recent call last):\n ...\n ValueError: File/directory does not exist: /no/such/path\n \"\"\"\n if isinstance(path, unicode):\n path = path.encode('utf-8')\n path = os.path.expanduser(path)\n if not os.path.exists(path):\n raise ValueError(_('File/directory does not exist: %s') % path)\n return os.path.abspath(path)\n\n\ndef WebRootCheck(path):\n \"\"\"\n Verify that a string is a valid web root path, normalize the slashes.\n\n >>> WebRootCheck('/')\n ''\n\n >>> WebRootCheck('/foo//bar////baz//')\n '/foo/bar/baz'\n\n >>> WebRootCheck('/foo/$%!')\n Traceback (most recent call last):\n ...\n ValueError: Invalid web root: /foo/$%!\n \"\"\"\n p = re.sub('/+', '/', '/%s/' % path)[:-1]\n if (p != CleanText(p, banned=CleanText.NONPATH).clean):\n raise ValueError('Invalid web root: %s' % path)\n return p\n\n\ndef FileCheck(path=None):\n \"\"\"\n Verify that a string is a valid path to a file, make it absolute.\n\n >>> FileCheck('/etc/../etc/passwd')\n '/etc/passwd'\n\n >>> FileCheck('/')\n Traceback (most recent call last):\n ...\n ValueError: Not a file: /\n \"\"\"\n if path in (None, 'None', 'none', ''):\n return None\n path = PathCheck(path)\n if not os.path.isfile(path):\n raise ValueError(_('Not a file: %s') % path)\n return path\n\n\ndef DirCheck(path=None):\n \"\"\"\n Verify that a string is a valid path to a directory, make it absolute.\n\n >>> DirCheck('/etc/../')\n '/'\n\n >>> DirCheck('/etc/passwd')\n Traceback (most recent call last):\n ...\n ValueError: Not a directory: /etc/passwd\n \"\"\"\n if path in (None, 'None', 'none', ''):\n return None\n path = PathCheck(path)\n if not os.path.isdir(path):\n raise ValueError(_('Not a directory: %s') % path)\n return path\n\n\ndef NewPathCheck(path):\n \"\"\"\n Verify that a string is a valid path to a directory, make it absolute.\n\n >>> NewPathCheck('/magic')\n '/magic'\n\n >>> NewPathCheck('/no/such/path/magic')\n Traceback (most recent call last):\n ...\n ValueError: File/directory does not exist: /no/such/path\n \"\"\"\n PathCheck(os.path.dirname(path))\n return os.path.abspath(path)\n\ndef UrlCheck(url):\n \"\"\"\n Verify that a url parsed string has a valid uri scheme\n\n >>> UrlCheck(\"http://mysite.com\")\n 'http://mysite.com'\n\n >>> UrlCheck(\"/not-valid.net\")\n Traceback (most recent call last):\n ...\n ValueError: Not a valid url: ...\n\n >>> UrlCheck(\"tallnet://some-host.com\")\n Traceback (most recent call last):\n ...\n ValueError: Not a valid url: tallnet://some-host.com\n \"\"\"\n uri = urlparse(url)\n if not uri.scheme in URI_SCHEMES:\n raise ValueError(_(\"Not a valid url: %s\") % url)\n else:\n return url\n\ndef EmailCheck(email):\n \"\"\"\n Verify that a string is a valid email\n\n >>> EmailCheck(\"test@test.com\")\n 'test@test.com'\n \"\"\"\n if not EMAIL_RE.match(email):\n raise ValueError(_(\"Not a valid e-mail: %s\") % email)\n return email\n\n\ndef GPGKeyCheck(value):\n \"\"\"\n Strip a GPG fingerprint of all spaces, make sure it seems valid.\n Will also accept e-mail addresses, for legacy reasons.\n\n >>> GPGKeyCheck('User@Foo.com')\n 'User@Foo.com'\n\n >>> GPGKeyCheck('1234 5678 abcd EF00')\n '12345678ABCDEF00'\n\n >>> GPGKeyCheck('12345678')\n '12345678'\n\n >>> GPGKeyCheck('B906 EA4B 8A28 15C4 F859 6F9F 47C1 3F3F ED73 5179')\n 'B906EA4B8A2815C4F8596F9F47C13F3FED735179'\n\n >>> GPGKeyCheck('B906 8A28 15C4 F859 6F9F 47C1 3F3F ED73 5179')\n Traceback (most recent call last):\n ...\n ValueError: Not a GPG key ID or fingerprint\n\n >>> GPGKeyCheck('B906 8X28 1111 15C4 F859 6F9F 47C1 3F3F ED73 5179')\n Traceback (most recent call last):\n ...\n ValueError: Not a GPG key ID or fingerprint\n \"\"\"\n value = value.replace(' ', '').replace('\\t', '').strip()\n if value in ('!CREATE', '!PASSWORD'):\n return value\n try:\n if len(value) not in (8, 16, 40):\n raise ValueError(_('Not a GPG key ID or fingerprint'))\n if re.match(r'^[0-9A-F]+$', value.upper()) is None:\n raise ValueError(_('Not a GPG key ID or fingerprint'))\n except ValueError:\n try:\n return EmailCheck(value)\n except ValueError:\n raise ValueError(_('Not a GPG key ID or fingerprint'))\n return value.upper()\n\n\nclass IgnoreValue(Exception):\n pass\n\n\ndef IgnoreCheck(data):\n raise IgnoreValue()\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n result = doctest.testmod(optionflags=doctest.ELLIPSIS)\n print('%s' % (result, ))\n if result.failed:\n sys.exit(1)\n" }, { "path": "mailpile/conn_brokers.py", "content": "from __future__ import print_function\n# Connection brokers facilitate & manage incoming and outgoing connections.\n#\n# The idea is that code actually tells us what it wants to do, so we can\n# choose an appropriate mechanism for connecting or receiving incoming\n# connections.\n#\n# Libraries which use socket.create_connection can be monkey-patched\n# to use a broker on a connection-by-connection bases like so:\n#\n# with broker.context(need=[broker.OUTGOING_CLEARTEXT,\n# broker.OUTGOING_SMTP]) as ctx:\n# conn = somelib.connect(something)\n# print 'Connected with encryption: %s' % ctx.encryption\n#\n# The context variable will then contain metadata about what sort of\n# connection was made.\n#\n# See the Capability class below for a list of attributes that can be\n# used to describe an outgoing (or incoming) connection.\n#\n# In particular, using the master broker will implement a prioritised\n# connection strategy where the most secure options are tried first and\n# things gracefully degrade. Protocols like IMAP, SMTP or POP3 will be\n# transparently upgraded to use STARTTLS.\n#\n# TODO:\n# - Implement a TorBroker\n# - Implement a PageKiteBroker\n# - Implement HTTP/SMTP/IMAP/POP3 TLS upgrade-brokers\n# - Prevent unbrokered socket.socket connections\n#\nimport datetime\nimport socket\nimport ssl\nimport subprocess\nimport sys\nimport threading\nimport time\nimport traceback\n\ntry:\n import cryptography\n import cryptography.hazmat.backends\n import cryptography.hazmat.primitives.hashes\n try:\n import cryptography.x509 as cryptography_x509\n except ImportError:\n cryptography_x509 = None\nexcept ImportError:\n cryptography = None\n\n# Import SOCKS proxy support...\ntry:\n import sockschain as socks\nexcept ImportError:\n try:\n import socks\n except ImportError:\n socks = None\n\nimport mailpile.security as security\nfrom mailpile.i18n import gettext\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.commands import Command\nfrom mailpile.util import md5_hex, dict_merge, monkey_patch\nfrom mailpile.security import tls_sock_cert_sha256\n\n\n_ = lambda s: s\n\nKNOWN_ONION_MAP = {\n 'www.mailpile.is': 'clgs64523yi2bkhz.onion'\n}\n\n\noriginal_scc = socket.create_connection\nmonkey_clean_scc = socket.create_connection\nmonkey_thread_local = threading.local()\n\n\ndef MonkeySockCreateConn(*args, **kwargs):\n if hasattr(monkey_thread_local, 'scc'):\n cconn = (monkey_thread_local.scc or [monkey_clean_scc])[-1]\n else:\n cconn = monkey_clean_scc\n return cconn(*args, **kwargs)\n\n\ndef _explain_encryption(sock):\n try:\n algo, proto, bits = sock.cipher()\n return (\n _('%(tls_version)s (%(bits)s bit %(algorithm)s)')\n ) % {\n 'bits': bits,\n 'tls_version': proto,\n 'algorithm': algo}\n except (ValueError, AttributeError):\n return _('no encryption')\n\n\nclass Capability(object):\n \"\"\"\n These are constants defining different types of outgoing or incoming\n connections. Brokers use these to describe what sort of connections they\n are capable of handling, and calling code uses these to describe the\n intent of network connection.\n \"\"\"\n OUTGOING_RAW = 'o:raw' # Request this to avoid meddling brokers\n OUTGOING_ENCRYPTED = 'o:e' # Request this if sending encrypted data\n OUTGOING_CLEARTEXT = 'o:c' # Request this if sending clear-text data\n OUTGOING_TRACKABLE = 'o:t' # Reject this to require anonymity\n OUTGOING_SMTP = 'o:smtp' # These inform brokers what protocol is being\n OUTGOING_IMAP = 'o:imap' # .. used, to allow protocol-specific features\n OUTGOING_POP3 = 'o:pop3' # .. such as enabling STARTTLS or upgrading\n OUTGOING_HTTP = 'o:http' # .. HTTP to HTTPS.\n OUTGOING_HTTPS = 'o:https' # ..\n OUTGOING_SMTPS = 'o:smtps' # ..\n OUTGOING_POP3S = 'o:pop3s' # ..\n OUTGOING_IMAPS = 'o:imaps' # ..\n\n INCOMING_RAW = 20\n INCOMING_LOCALNET = 21\n INCOMING_INTERNET = 22\n INCOMING_DARKNET = 23\n INCOMING_SMTP = 24\n INCOMING_IMAP = 25\n INCOMING_POP3 = 26\n INCOMING_HTTP = 27\n INCOMING_HTTPS = 28\n\n ALL_OUTGOING = set([OUTGOING_RAW, OUTGOING_ENCRYPTED, OUTGOING_CLEARTEXT,\n OUTGOING_TRACKABLE,\n OUTGOING_SMTP, OUTGOING_IMAP, OUTGOING_POP3,\n OUTGOING_SMTPS, OUTGOING_IMAPS, OUTGOING_POP3S,\n OUTGOING_HTTP, OUTGOING_HTTPS])\n\n ALL_OUTGOING_ENCRYPTED = set([OUTGOING_RAW, OUTGOING_TRACKABLE,\n OUTGOING_ENCRYPTED,\n OUTGOING_HTTPS, OUTGOING_SMTPS,\n OUTGOING_POP3S, OUTGOING_IMAPS])\n\n ALL_INCOMING = set([INCOMING_RAW, INCOMING_LOCALNET, INCOMING_INTERNET,\n INCOMING_DARKNET, INCOMING_SMTP, INCOMING_IMAP,\n INCOMING_POP3, INCOMING_HTTP, INCOMING_HTTPS])\n\n\nclass CapabilityFailure(IOError):\n \"\"\"\n This exception is raised when capability requirements can't be satisfied.\n It extends the IOError, so unaware code just thinks the network is lame.\n\n >>> try:\n ... raise CapabilityFailure('boo')\n ... except IOError:\n ... print('ok')\n ok\n \"\"\"\n pass\n\n\nclass Url(str):\n def __init__(self, *args, **kwargs):\n str.__init__(self, *args, **kwargs)\n self.encryption = None\n self.anonymity = None\n self.on_internet = False\n self.on_localnet = False\n self.on_darknet = None\n\n\nclass BrokeredContext(object):\n \"\"\"\n This is the context returned by the BaseConnectionBroker.context()\n method. It takes care of monkey-patching the socket.create_connection\n method and then cleaning the mess up afterwards, and collecting metadata\n from the brokers describing what sort of connection was established.\n\n WARNING: In spite of our best efforts (locking, etc.), mixing brokered\n and unbrokered code will not work well at all. The patching\n approach also limits us to initiating one outgoing connection\n at a time.\n \"\"\"\n def __init__(self, broker, need=None, reject=None, oneshot=False):\n self._broker = broker\n self._need = need\n self._reject = reject\n self._oneshot = oneshot\n self._reset()\n\n def __str__(self):\n hostport = '%s:%s' % (self.address or ('unknown', 'none'))\n if self.error:\n return _('Failed to connect to %s: %s') % (hostport, self.error)\n\n if self.anonymity:\n network = self.anonymity\n elif self.on_darknet:\n network = self.on_darknet\n elif self.on_localnet:\n network = _('the local network')\n elif self.on_internet:\n network = _('the Internet')\n else:\n return _('Attempting to connect to %(host)s') % {'host': hostport}\n\n return _('Connected to %(host)s over %(network)s with %(encryption)s.'\n ) % {\n 'network': network,\n 'host': hostport,\n 'encryption': self.encryption or _('no encryption')\n }\n\n def _reset(self):\n self.error = None\n self.address = None\n self.encryption = None\n self.anonymity = None\n self.on_internet = False\n self.on_localnet = False\n self.on_darknet = None\n\n def __enter__(self, *args, **kwargs):\n def create_brokered_conn(address, *a, **kw):\n self._reset()\n return self._broker.create_conn_with_caps(\n address, self, self._need, self._reject, *a, **kw)\n\n if hasattr(monkey_thread_local, 'scc'):\n monkey_thread_local.scc.append(create_brokered_conn)\n else:\n monkey_thread_local.scc = [create_brokered_conn]\n if socket.create_connection != MonkeySockCreateConn:\n socket.create_connection = MonkeySockCreateConn\n\n return self\n\n def __exit__(self, *args, **kwargs):\n monkey_thread_local.scc.pop(-1)\n\n\nclass BaseConnectionBroker(Capability):\n \"\"\"\n This is common code used by most of the connection brokers.\n \"\"\"\n SUPPORTS = []\n\n def __init__(self, master=None):\n self.supports = list(self.SUPPORTS)[:]\n self.master = master\n self._config = None\n self._debug = master._debug if (master is not None) else None\n\n def configure(self):\n self.supports = list(self.SUPPORTS)[:]\n\n def set_config(self, config):\n self._config = config\n self.configure()\n\n def config(self):\n if self._config is not None:\n return self._config\n if self.master is not None:\n return self.master.config()\n return None\n\n def _raise_or_none(self, exc, why):\n if exc is not None:\n raise exc(why)\n return None\n\n def _check(self, need, reject, _raise=CapabilityFailure):\n for n in need or []:\n if n not in self.supports:\n if self._debug is not None:\n self._debug('%s: lacking capabilty %s' % (self, n))\n return self._raise_or_none(_raise, 'Lacking %s' % n)\n for n in reject or []:\n if n in self.supports:\n if self._debug is not None:\n self._debug('%s: unwanted capabilty %s' % (self, n))\n return self._raise_or_none(_raise, 'Unwanted %s' % n)\n if self._debug is not None:\n self._debug('%s: checks passed!' % (self, ))\n return self\n\n def _describe(self, context, conn):\n return conn\n\n def debug(self, val):\n self._debug = val\n return self\n\n def context(self, need=None, reject=None, oneshot=False):\n return BrokeredContext(self, need=need, reject=reject, oneshot=oneshot)\n\n def create_conn_with_caps(self, address, context, need, reject,\n *args, **kwargs):\n if context.address is None:\n context.address = address\n conn = self._check(need, reject)._create_connection(context, address,\n *args, **kwargs)\n return self._describe(context, conn)\n\n def create_connection(self, address, *args, **kwargs):\n n = kwargs.get('need', None)\n r = kwargs.get('reject', None)\n c = kwargs.get('context', None)\n for kw in ('need', 'reject', 'context'):\n if kw in kwargs:\n del kwargs[kw]\n return self.create_conn_with_caps(address, c, n, r, *args, **kwargs)\n\n # Should implement socket.create_connection or an equivalent.\n # Context, if not None, should be informed with metadata about the\n # connection.\n def _create_connection(self, context, address, *args, **kwargs):\n raise NotImplementedError('Subclasses override this')\n\n def get_urls(self, listening_fd,\n need=None, reject=None, **kwargs):\n try:\n return self._check(need, reject)._get_urls(listening_fd, **kwargs)\n except CapabilityFailure:\n return []\n\n # Returns a list of Url objects for this listener\n def _get_urls(self, listening_fd,\n proto=None, username=None, password=None):\n raise NotImplementedError('Subclasses override this')\n\n\nclass TcpConnectionBroker(BaseConnectionBroker):\n \"\"\"\n The basic raw TCP/IP connection broker.\n\n The only clever thing this class does, is to avoid trying to connect\n to .onion addresses, preventing that from leaking over DNS.\n \"\"\"\n SUPPORTS = (\n # Normal TCP/IP is not anonymous, and we do not have incoming\n # capability unless we have a public IP.\n (Capability.ALL_OUTGOING) |\n (Capability.ALL_INCOMING - set([Capability.INCOMING_INTERNET]))\n )\n LOCAL_NETWORKS = ['localhost', '127.0.0.1', '::1']\n FIXED_NO_PROXY_LIST = ['localhost', '127.0.0.1', '::1']\n DEBUG_FMT = '%s: Raw TCP conn to: %s'\n\n def configure(self):\n BaseConnectionBroker.configure(self)\n # FIXME: If our config indicates we have a public IP, add the\n # INCOMING_INTERNET capability.\n\n def _describe(self, context, conn):\n try:\n (host, port) = conn.getpeername()[:2]\n if host.lower() in self.LOCAL_NETWORKS:\n context.on_localnet = True\n else:\n context.on_internet = True\n except TypeError:\n # conn.getpeername() may return None\n pass\n context.encryption = None\n return conn\n\n def _in_no_proxy_list(self, address):\n cfg_no_proxy = self.config().get_proxy_settings().get('no_proxy', '')\n no_proxy = (self.FIXED_NO_PROXY_LIST +\n [a.lower().strip() for a in cfg_no_proxy.split(',')])\n return (address[0].lower() in no_proxy)\n\n def _avoid(self, address):\n proxy_settings = self.config().get_proxy_settings()\n if (proxy_settings['protocol'] not in ('none', 'unknown', 'system')\n and not proxy_settings.get('fallback', False)\n and not self._in_no_proxy_list(address)):\n raise CapabilityFailure('Proxy fallback is disabled')\n\n def _broker_avoid(self, address):\n if address[0].endswith('.onion'):\n raise CapabilityFailure('Cannot connect to .onion addresses')\n\n def _conn(self, address, *args, **kwargs):\n clean_kwargs = dict((k, v) for k, v in kwargs.iteritems()\n if not k.startswith('_'))\n return original_scc(address, *args, **clean_kwargs)\n\n def _create_connection(self, context, address, *args, **kwargs):\n self._avoid(address)\n self._broker_avoid(address)\n if self._debug is not None:\n self._debug(self.DEBUG_FMT % (self, address))\n return self._conn(address, *args, **kwargs)\n\n\nclass SocksConnBroker(TcpConnectionBroker):\n \"\"\"\n This broker offers the same services as the TcpConnBroker, but over a\n SOCKS connection.\n \"\"\"\n SUPPORTS = []\n CONFIGURED = Capability.ALL_OUTGOING\n PROXY_TYPES = ('socks5', 'http', 'socks4')\n DEFAULT_PROTO = 'socks5'\n\n DEBUG_FMT = '%s: Raw SOCKS5 conn to: %s'\n IOERROR_FMT = _('SOCKS error, %s')\n IOERROR_MSG = {\n 'timed out': _('timed out'),\n 'Host unreachable': _('host unreachable'),\n 'Connection refused': _('connection refused')\n }\n\n def _describe(self, context, conn):\n context.on_darknet = ('proxy (%s:%d)'\n % (self.proxy_config['host'], self.proxy_config['port']))\n return conn\n\n def __init__(self, *args, **kwargs):\n TcpConnectionBroker.__init__(self, *args, **kwargs)\n self.proxy_config = None\n self.typemap = {}\n\n def configure(self):\n BaseConnectionBroker.configure(self)\n proxy_settings = self.config().get_proxy_settings()\n if proxy_settings.get('protocol') in self.PROXY_TYPES:\n self.proxy_config = proxy_settings\n self.supports = list(self.CONFIGURED)[:]\n self.typemap = {\n 'socks5': socks.PROXY_TYPE_SOCKS5,\n 'socks4': socks.PROXY_TYPE_SOCKS4,\n 'tor': socks.PROXY_TYPE_SOCKS5, # For TorConnBroker\n 'tor-risky': socks.PROXY_TYPE_SOCKS5, # For TorConnBroker\n 'http': socks.PROXY_TYPE_HTTP}\n else:\n self.proxy_config = None\n self.supports = []\n\n def _auth_args(self):\n return {\n 'username': self.proxy_config.get('username') or None,\n 'password': self.proxy_config.get('password') or None}\n\n def _avoid(self, address):\n if self._in_no_proxy_list(address):\n raise CapabilityFailure('Proxy to %s:%s disabled by policy'\n ) % address\n\n def _fix_address_tuple(self, address):\n return (str(address[0]), int(address[1]))\n\n def _conn(self, address, timeout=None, source_address=None, **kwargs):\n sock = socks.socksocket()\n proxytype = self.typemap.get(self.proxy_config.get('protocol'),\n self.typemap[self.DEFAULT_PROTO])\n sock.setproxy(proxytype=proxytype,\n addr=self.proxy_config.get('host'),\n port=int(self.proxy_config.get('port', 0)),\n rdns=True,\n **self._auth_args())\n if timeout and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:\n sock.settimeout(float(timeout))\n if source_address:\n raise CapabilityFailure('Cannot bind source address')\n try:\n address = self._fix_address_tuple(address)\n sock.connect(address)\n except socks.ProxyError as e:\n if self._debug is not None:\n self._debug(traceback.format_exc())\n code, msg = e.message\n raise IOError(_(self.IOERROR_FMT\n ) % (_(self.IOERROR_MSG.get(msg, msg)), ))\n return sock\n\n\nclass TorConnBroker(SocksConnBroker):\n \"\"\"\n This broker offers the same services as the TcpConnBroker, but over Tor.\n\n This removes the \"trackable\" capability, so requests that reject it can\n find their way here safely...\n\n This broker only volunteers to carry encrypted traffic, because Tor\n exit nodes may be hostile.\n \"\"\"\n SUPPORTS = []\n CONFIGURED = (Capability.ALL_OUTGOING_ENCRYPTED\n - set([Capability.OUTGOING_TRACKABLE]))\n REJECTS = None\n PROXY_TYPES = ('tor', )\n DEFAULT_PROTO = 'tor'\n\n DEBUG_FMT = '%s: Raw Tor conn to: %s'\n IOERROR_FMT = _('Tor error, %s')\n IOERROR_MSG = dict_merge(SocksConnBroker.IOERROR_MSG, {\n 'bad input': _('connection refused') # FIXME: Is this right?\n })\n\n def _describe(self, context, conn):\n context.on_darknet = 'Tor'\n context.anonymity = 'Tor'\n return conn\n\n def _auth_args(self):\n # FIXME: Tor uses the auth information as a signal to change\n # circuits. We may have use for this at some point.\n return {}\n\n def _fix_address_tuple(self, address):\n host = str(address[0])\n return (KNOWN_ONION_MAP.get(host.lower(), host), int(address[1]))\n\n def _broker_avoid(self, address):\n # Disable the avoiding of .onion addresses added above\n pass\n\n\nclass TorRiskyBroker(TorConnBroker):\n \"\"\"\n This differs from the TorConnBroker in that it will allow \"cleartext\"\n traffic to anywhere - this is dangerous, because exit nodes could mess\n with our traffic.\n \"\"\"\n CONFIGURED = (Capability.ALL_OUTGOING\n - set([Capability.OUTGOING_TRACKABLE]))\n DEBUG_FMT = '%s: Risky Tor conn to: %s'\n PROXY_TYPES = ('tor-risky', )\n DEFAULT_PROTO = 'tor-risky'\n\n\nclass TorOnionBroker(TorConnBroker):\n \"\"\"\n This broker offers the same services as the TcpConnBroker, but over Tor.\n\n This removes the \"trackable\" capability, so requests that reject it can\n find their way here safely...\n\n This differs from the TorConnBroker in that it will allow \"cleartext\"\n traffic, since we trust the traffic never leaves the Tor network and\n we don't have hostile exits to worry about.\n \"\"\"\n SUPPORTS = []\n CONFIGURED = (Capability.ALL_OUTGOING\n - set([Capability.OUTGOING_TRACKABLE]))\n REJECTS = None\n DEBUG_FMT = '%s: Tor onion conn to: %s'\n PROXY_TYPES = ('tor', 'tor-risky')\n\n def _broker_avoid(self, address):\n host = KNOWN_ONION_MAP.get(address[0], address[0])\n if not host.endswith('.onion'):\n raise CapabilityFailure('Can only connect to .onion addresses')\n\n\nclass BaseConnectionBrokerProxy(TcpConnectionBroker):\n \"\"\"\n Brokers based on this establish a RAW connection and then manipulate it\n in some way, generally to implement proxying or TLS wrapping.\n \"\"\"\n SUPPORTS = []\n WANTS = [Capability.OUTGOING_RAW]\n REJECTS = None\n\n def _proxy_address(self, address):\n return address\n\n def _proxy(self, conn):\n raise NotImplementedError('Subclasses override this')\n\n def _wrap_ssl(self, conn):\n if self._debug is not None:\n self._debug('%s: Wrapping socket with SSL' % (self, ))\n return ssl.wrap_socket(conn)\n\n def _create_connection(self, context, address, *args, **kwargs):\n address = self._proxy_address(address)\n if self.master:\n conn = self.master.create_conn_with_caps(\n address, context, self.WANTS, self.REJECTS, *args, **kwargs)\n else:\n conn = TcpConnectionBroker._create_connection(self, context,\n address,\n *args, **kwargs)\n return self._proxy(conn)\n\n\nclass AutoTlsConnBroker(BaseConnectionBrokerProxy):\n \"\"\"\n This broker tries to auto-upgrade connections to use TLS, or at\n least do the SSL handshake here so we can record info about it.\n \"\"\"\n SUPPORTS = [Capability.OUTGOING_HTTP, Capability.OUTGOING_HTTPS,\n Capability.OUTGOING_IMAPS, Capability.OUTGOING_SMTPS,\n Capability.OUTGOING_POP3S]\n WANTS = [Capability.OUTGOING_RAW, Capability.OUTGOING_ENCRYPTED]\n\n def _describe(self, context, conn):\n context.encryption = _explain_encryption(conn)\n return conn\n\n def _proxy_address(self, address):\n if address[0].endswith('.onion'):\n raise CapabilityFailure('I do not like .onion addresses')\n if int(address[1]) != 443:\n # FIXME: Import HTTPS Everywhere database to make this work?\n raise CapabilityFailure('Not breaking clear-text HTTP yet')\n return address\n\n def _proxy(self, conn):\n return self._wrap_ssl(conn)\n\n\nclass AutoSmtpStartTLSConnBroker(BaseConnectionBrokerProxy):\n pass\n\n\nclass AutoImapStartTLSConnBroker(BaseConnectionBrokerProxy):\n pass\n\n\nclass AutoPop3StartTLSConnBroker(BaseConnectionBrokerProxy):\n pass\n\n\nclass MasterBroker(BaseConnectionBroker):\n \"\"\"\n This is the master broker. It implements a prioritised list of\n connection brokers, each of which is tried in turn until a match\n is found. As such, more secure brokers should register themselves\n with a higher priority - if they fail, we fall back to less\n secure connection strategies.\n \"\"\"\n def __init__(self, *args, **kwargs):\n BaseConnectionBroker.__init__(self, *args, **kwargs)\n self.brokers = []\n self.history = []\n self._debug = self._debugger\n self.debug_callback = None\n\n def configure(self):\n for prio, cb in self.brokers:\n cb.configure()\n\n def _debugger(self, *args, **kwargs):\n if self.debug_callback is not None:\n self.debug_callback(*args, **kwargs)\n\n def register_broker(self, priority, cb):\n \"\"\"\n Brokers should register themselves with priorities as follows:\n\n - 1000-1999: Content-agnostic raw connections\n - 3000-3999: Secure network layers: VPNs, Tor, I2P, ...\n - 5000-5999: Proxies required to reach the wider Internet\n - 7000-7999: Protocol enhancments (non-security related)\n - 9000-9999: Security-related protocol enhancements\n\n \"\"\"\n self.brokers.append((priority, cb(master=self)))\n self.brokers.sort()\n self.brokers.reverse()\n\n def get_fd_context(self, fileno):\n for t, fd, context in reversed(self.history):\n if fd == fileno:\n return context\n return BrokeredContext(self)\n\n def create_conn_with_caps(self, address, context, need, reject,\n *args, **kwargs):\n history_event = kwargs.get('_history_event')\n if history_event is None:\n history_event = [int(time.time()), None, context]\n self.history = self.history[-50:]\n self.history.append(history_event)\n kwargs['_history_event'] = history_event\n else:\n history_event[-1] = context\n\n if context.address is None:\n context.address = address\n\n et = v = t = None\n for prio, cb in self.brokers:\n try:\n conn = cb.debug(self._debug).create_conn_with_caps(\n address, context, need, reject, *args, **kwargs)\n if conn:\n history_event[1] = conn.fileno()\n return conn\n except (CapabilityFailure, NotImplementedError):\n # These are internal; we assume they're already logged\n # for debugging but don't bother the user with them.\n pass\n except:\n et, v, t = sys.exc_info()\n if et is not None:\n context.error = '%s' % v\n raise et, v, t\n\n context.error = _('No connection method found')\n raise CapabilityFailure(context.error)\n\n def get_urls(self, listening_fd, need=None, reject=None):\n urls = []\n for prio, cb in self.brokers:\n urls.extend(cb.debug(self._debug).get_urls(listening_fd))\n return urls\n\n\ndef DisableUnbrokeredConnections():\n \"\"\"Enforce the use of brokers EVERYWHERE!\"\"\"\n def CreateConnWarning(*args, **kwargs):\n print('*** socket.create_connection used without a broker ***')\n traceback.print_stack()\n raise IOError('FIXME: Please use within a broker context')\n monkey_clean_scc = CreateConnWarning\n socket.create_connection = CreateConnWarning\n\n\nclass NetworkHistory(Command):\n \"\"\"Show recent network history\"\"\"\n SYNOPSIS = (None, 'logs/network', 'logs/network', None)\n ORDER = ('Internals', 6)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n if self.result:\n def fmt(result):\n dt = datetime.datetime.fromtimestamp(result[0])\n return '%2.2d:%2.2d %s' % (dt.hour, dt.minute, result[-1])\n return '\\n'.join(fmt(r) for r in self.result)\n return _('No network events recorded')\n\n def command(self):\n return self._success(_('Listed recent network events'),\n result=Master.history)\n\n\nclass GetTlsCertificate(Command):\n \"\"\"Fetch and parse a server's TLS certificate\"\"\"\n SYNOPSIS = (None, 'crypto/tls/getcert', 'crypto/tls/getcert', '[--tofu-save|--tofu-clear]')\n ORDER = ('Internals', 6)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = {\n 'tofu-clear': 'Remove from TOFU certificate store',\n 'tofu-save': 'Save to our TOFU certificate store',\n 'host': 'Name of remote server'\n }\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n if self.result:\n def fmt(h, r):\n return '%s:\\t%s' % (h, r[-1] or r[1])\n return '\\n'.join(fmt(h, r) for h, r in self.result.iteritems())\n return _('No certificates found')\n\n def command(self):\n if self.data.get('_method', 'POST') != 'POST':\n # Allow HTTP GET as a no-op, so the user can see a friendly form.\n return self._success(_('Examine TLS certificates'))\n\n config = self.session.config\n tofu_save = self.data.get('tofu-save', '--tofu-save' in self.args)\n tofu_clear = self.data.get('tofu-clear', '--tofu-clear' in self.args)\n hosts = (list(s for s in self.args if not s.startswith('--')) +\n self.data.get('host', []))\n\n def ts(t):\n return int(time.mktime(t.timetuple()))\n\n def oidName(oid):\n return {\n '2.5.4.3': 'commonName',\n '2.5.4.4': 'surname',\n '2.5.4.5': 'serialNumber',\n '2.5.4.6': 'countryName',\n '2.5.4.7': 'localityName',\n '2.5.4.8': 'stateOrProvinceName',\n '2.5.4.9': 'streetAddress',\n '2.5.4.10': 'organizationName',\n '2.5.4.11': 'organizationalUnitName'\n }.get(oid.dotted_string,\n getattr(oid, '_name', oid.dotted_string))\n\n def oidmap(entries):\n return dict((oidName(e.oid), e.value) for e in entries)\n\n def subjmap(stext):\n def subjpair(kv):\n k, v = kv.split('=', 1)\n return ({'CN': 'commonName',\n 'C': 'countryName',\n 'ST': 'stateOrProvinceName',\n 'L': 'localityName',\n 'O': 'organizationName',\n 'OU': 'organizationalUnitName'}.get(k, k), v)\n parts = []\n for part in stext.strip().split('/'):\n if '=' in part:\n parts.append(part)\n elif parts:\n parts[-1] += '/' + part\n return dict(subjpair(kv) for kv in parts)\n\n def fingerprint(cert_sha_256):\n fp = ['%2.2x' % ord(b) for b in cert_sha_256]\n fp2 = [fp[i*2] + fp[i*2 + 1] for i in range(0, len(fp)/2)]\n return fp2\n\n def pts(t):\n dt, tz = t.rsplit(' ', 1) # Strip off the timezone\n return datetime.datetime.strptime(dt, '%b %d %H:%M:%S %Y')\n\n def parse_pem_cert(cert_pem, s256):\n cert_sha_256 = s256.decode('base64')\n now = datetime.datetime.today()\n if cryptography_x509 is None:\n # Shell out to openssl, boo.\n (stdout, stderr) = subprocess.Popen(\n ['openssl', 'x509',\n '-subject', '-issuer', '-dates', '-noout'],\n stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n stdin=subprocess.PIPE).communicate(input=str(cert_pem))\n if not stdout:\n raise ValueError(stderr)\n details = dict(l.split('=', 1)\n for l in stdout.strip().splitlines()\n if l and '=' in l)\n details['notAfter'] = pts(details['notAfter'])\n details['notBefore'] = pts(details['notBefore'])\n return {\n 'fingerprint': fingerprint(cert_sha_256),\n 'date_matches': False,\n 'date_matches': ((details['notBefore'] < now) and\n (details['notAfter'] > now)),\n 'not_valid_before': ts(details['notBefore']),\n 'not_valid_after': ts(details['notAfter']),\n 'subject': subjmap(details['subject']),\n 'issuer': subjmap(details['issuer'])}\n else:\n parsed = cryptography_x509.load_pem_x509_certificate(\n str(cert_pem),\n cryptography.hazmat.backends.default_backend())\n return {\n 'fingerprint': fingerprint(cert_sha_256),\n 'date_matches': ((parsed.not_valid_before < now) and\n (parsed.not_valid_after > now)),\n 'not_valid_before': ts(parsed.not_valid_before),\n 'not_valid_after': ts(parsed.not_valid_after),\n 'subject': oidmap(parsed.subject),\n 'issuer': oidmap(parsed.issuer)}\n\n def attempt_starttls(addr, sock):\n # Attempt a minimal SMTP interaction, for STARTTLS support\n\n # We attempt a non-blocking peek unless we're sure this is\n # a port normally used for clear-text SMTP.\n peeking = int(addr[1]) not in (25, 587, 143)\n\n # If this isn't a known TLS port, then we sleep a bit to give a\n # greeting time to arrive.\n if peeking and int(addr[1]) not in (443, 465, 993, 995):\n time.sleep(0.4)\n\n try:\n # Look for an SMTP (or IMAP) greeting\n if peeking:\n sock.setblocking(0)\n # Note: This will throw a TypeError if we are connected\n # over Tor (or other SOCKS).\n first = sock.recv(1024, socket.MSG_PEEK) or ''\n else:\n sock.settimeout(10)\n first = sock.recv(1024) or ''\n\n if first[:4] == '220 ':\n # This is an SMTP greeting\n if peeking:\n sock.setblocking(1)\n sock.recv(1024)\n sock.sendall('EHLO example.com\\r\\n')\n if (sock.recv(1024) or '')[:1] == '2':\n sock.sendall('STARTTLS\\r\\n')\n sock.recv(1024)\n\n elif first[:4] == '* OK':\n # This is an IMAP4 greeting\n if peeking:\n sock.setblocking(1)\n sock.recv(1024)\n sock.sendall('* STARTTLS\\r\\n')\n sock.recv(1024)\n\n except (TypeError, IOError, OSError):\n pass\n finally:\n sock.setblocking(1)\n\n certs = {}\n ok = changes = 0\n for host in hosts:\n try:\n addr = host.replace(' ', '').split(':') + ['443']\n addr = (addr[0], int(addr[1]))\n\n try:\n with Master.context(need=[Master.OUTGOING_ENCRYPTED,\n Master.OUTGOING_RAW]) as ctx:\n sock = socket.create_connection(addr, timeout=30)\n attempt_starttls(addr, sock)\n ssls = ssl.wrap_socket(sock, use_web_ca=True, tofu=False)\n hostname_matches = True\n cert_validated = True\n\n except (ssl.SSLError, ssl.CertificateError) as e:\n if isinstance(e, ssl.CertificateError):\n cert_validated = True\n hostname_matches = False\n else:\n cert_validated = False\n hostname_matches = 'unknown'\n\n with Master.context(need=[Master.OUTGOING_ENCRYPTED,\n Master.OUTGOING_RAW]) as ctx:\n sock = socket.create_connection(addr, timeout=30)\n attempt_starttls(addr, sock)\n ssls = ssl.wrap_socket(sock, use_web_ca=False, tofu=False)\n\n cert = ssls.getpeercert(True)\n s256 = tls_sock_cert_sha256(cert=cert)\n ssls.close()\n\n cfg_key = md5_hex('%s:%d' % addr)\n if tofu_clear:\n if cfg_key in config.tls.keys():\n del config.tls[cfg_key]\n changes += 1\n if tofu_save:\n if cfg_key not in config.tls.keys():\n config.tls[cfg_key] = {'server': '%s:%d' % addr}\n cert_tofu = config.tls[cfg_key]\n cert_tofu.use_web_ca = False\n cert_tofu.accept_certs.append(s256)\n changes += 1\n else:\n cert_tofu = config.tls.get(cfg_key, {})\n\n tofu_seen = s256 in cert_tofu.get('accept_certs', [])\n using_tofu = not cert_tofu.get('use_web_ca', True)\n cert = {\n 'current_time': int(time.time()),\n 'cert_validated': cert_validated,\n 'hostname_matches': hostname_matches,\n 'tofu_seen': tofu_seen,\n 'using_tofu': using_tofu,\n 'tofu_invalid': (using_tofu and not tofu_seen),\n 'pem': ssl.DER_cert_to_PEM_cert(cert)}\n\n cert.update(parse_pem_cert(cert['pem'], s256))\n\n certs[host] = (True, s256, cert, None)\n ok += 1\n except Exception as e:\n certs[host] = (\n False, _('Failed to fetch certificate'), unicode(e),\n traceback.format_exc())\n\n if changes:\n self._background_save(config=True)\n\n if ok:\n return self._success(_('Downloaded TLS certificates'),\n result=certs)\n else:\n return self._error(_('Failed to download TLS certificates'),\n result=certs)\n\n\ndef SslWrapOnlyOnce(org_sslwrap, sock, *args, **kwargs):\n \"\"\"\n Since we like to wrap things our own way, this make ssl.wrap_socket\n into a no-op in the cases where we've alredy wrapped a socket.\n \"\"\"\n if not isinstance(sock, ssl.SSLSocket):\n ctx = Master.get_fd_context(sock.fileno())\n try:\n if 'server_hostname' not in kwargs:\n kwargs['server_hostname'] = ctx.address[0]\n sock = org_sslwrap(sock, *args, **kwargs)\n ctx.encryption = _explain_encryption(sock)\n except (socket.error, IOError, ssl.SSLError, ssl.CertificateError) as e:\n ctx.error = '%s' % e\n raise\n return sock\n\n\ndef SslContextWrapOnlyOnce(org_ctxwrap, self, sock, *args, **kwargs):\n return SslWrapOnlyOnce(\n lambda s, *a, **kwa: org_ctxwrap(self, s, *a, **kwa),\n sock, *args, **kwargs)\n\n\n_ = gettext\n\n\nif __name__ != \"__main__\":\n Master = MasterBroker()\n register = Master.register_broker\n register(1000, TcpConnectionBroker)\n register(9500, AutoTlsConnBroker)\n register(9500, AutoSmtpStartTLSConnBroker)\n register(9500, AutoImapStartTLSConnBroker)\n register(9500, AutoPop3StartTLSConnBroker)\n\n if socks is not None:\n register(1500, SocksConnBroker)\n register(3500, TorConnBroker)\n register(3500, TorRiskyBroker)\n register(3500, TorOnionBroker)\n\n # Note: At this point we have already imported security, which\n # also monkey-patches these same functions. This is a good\n # thing and is deliberate. :-)\n ssl.wrap_socket = monkey_patch(ssl.wrap_socket, SslWrapOnlyOnce)\n if hasattr(ssl, 'SSLContext'):\n ssl.SSLContext.wrap_socket = monkey_patch(\n ssl.SSLContext.wrap_socket, SslContextWrapOnlyOnce)\n\n from mailpile.plugins import PluginManager\n _plugins = PluginManager(builtin=__file__)\n _plugins.register_commands(NetworkHistory, GetTlsCertificate)\n\nelse:\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/crypto/__init__.py", "content": "" }, { "path": "mailpile/crypto/aes_utils.py", "content": "from __future__ import print_function\n# This is a compatibility wrapper for using whatever AES library is handy.\n# By default we support Cryptography and pyCrypto, with a preference for\n# Cryptography.\n\n# IMPORTANT:\n#\n# We currently only implement AES CTR mode, since this code is primarily\n# being used to write data to disk for long-term storage; the malleability\n# of CTR is considered a feature; if a bit gets flipped that doesn't destroy\n# all of the following blocks.\n#\n# This does mean we need to take special care with our IVs/nonces!\n#\nimport os\nimport struct\nfrom hashlib import md5\n\n\ndef make_cryptography_utils():\n import os\n import cryptography.hazmat.backends\n from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes\n\n def _aes_ctr(key, nonce):\n # Notes:\n #\n # The funky business with the prefixed nonce is because the first\n # iteration of this code used pycrypto's Crypto.Util.Counter with\n # the prefix argument. Cryptography doesn't have such a counter API,\n # but if we carefully set the nonce we can achieve compatibility.\n #\n # The MD5 digests save the caller from having to know our internal\n # size requirements; AES wants 128, we just mix all the bits we're\n # given. We expect the input to already be strongly random (so MD5's\n # weaknesses shouldn't matter), but it may of the wrong size.\n #\n hashed_key = md5(key).digest()\n prefixed_nonce = md5(nonce).digest()[:8] + '\\0\\0\\0\\0\\0\\0\\0\\1'\n return Cipher(\n algorithms.AES(hashed_key),\n modes.CTR(prefixed_nonce),\n backend=cryptography.hazmat.backends.default_backend())\n\n def aes_ctr_encryptor(key, nonce):\n return _aes_ctr(key, nonce).encryptor().update\n\n def aes_ctr_decryptor(key, nonce):\n return _aes_ctr(key, nonce).decryptor().update\n\n return aes_ctr_encryptor, aes_ctr_decryptor\n\n\ndef make_pycrypto_utils():\n from Crypto.Cipher import AES\n from Crypto.Util import Counter\n\n def _nonce_as_int(nonce):\n i1, i2, i3, i4 = struct.unpack(\">IIII\", nonce)\n return (i1 << 96 | i2 << 64 | i3 << 32 | i4)\n\n def _aes_ctr(key, nonce):\n # Notes:\n #\n # A previous iteration of this code used the Counter with a prefix,\n # which limited us to 2**64 iterations. This has been change to just\n # set an initial value and allow wraparound.\n #\n # The MD5 digests save the caller from having to know our internal\n # size requirements; AES wants 128, we just mix all the bits we're\n # given. We expect the input to already be strongly random (so MD5's\n # weaknesses shouldn't matter), but it may of the wrong size.\n #\n hashed_key = md5(key).digest()\n prefixed_nonce = md5(nonce).digest()[:8] + '\\0\\0\\0\\0\\0\\0\\0\\1'\n counter = Counter.new(128, initial_value=_nonce_as_int(prefixed_nonce))\n return AES.new(hashed_key, mode=AES.MODE_CTR, counter=counter)\n\n def aes_ctr_encryptor(key, nonce):\n return _aes_ctr(key, nonce).encrypt\n\n def aes_ctr_decryptor(key, nonce):\n return _aes_ctr(key, nonce).decrypt\n\n return aes_ctr_encryptor, aes_ctr_decryptor\n\n\ndef make_dummy_utils():\n def aes_ctr_encryptor(key):\n return lambda d: d\n\n def aes_ctr_decryptor(key):\n return lambda d: d\n\n return aes_ctr_encryptor, aes_ctr_decryptor\n\n\n##############################################################################\n\ntry:\n aes_ctr_encryptor, aes_ctr_decryptor = make_cryptography_utils()\nexcept ImportError:\n try:\n aes_ctr_encryptor, aes_ctr_decryptor = make_pycrypto_utils()\n except ImportError:\n raise ImportError(\"Please pip install cryptography (or pycrypto)\")\n\n\ndef getrandbits(count):\n bits = os.urandom(count // 8)\n rint = 0\n while bits:\n rint = (rint << 8) | struct.unpack(\"B\", bits[0])[0]\n bits = bits[1:]\n return rint\n\ndef aes_ctr_encrypt(key, iv, data):\n return aes_ctr_encryptor(key, iv)(data)\n\ndef aes_ctr_decrypt(key, iv, data):\n return aes_ctr_decryptor(key, iv)(data)\n\n\nif __name__ == \"__main__\":\n import base64\n\n bogus_key = \"01234567890abcdef\"\n bogus_nonce = \"this is a bogus nonce that is bogus\"\n hello = \"hello world\"\n\n results = []\n for name, backend in (('Cryptography', make_cryptography_utils),\n ('pyCrypto', make_pycrypto_utils)):\n aes_ctr_encryptor, aes_ctr_decryptor = backend()\n\n ct1 = aes_ctr_encryptor(bogus_key, bogus_nonce)(hello)\n results.append((name, base64.b64encode(ct1)))\n\n ct2 = aes_ctr_encrypt(bogus_key, bogus_nonce, hello)\n results.append((name, base64.b64encode(ct2)))\n\n assert(aes_ctr_decrypt(bogus_key, bogus_nonce, ct1) ==\n aes_ctr_decryptor(bogus_key, bogus_nonce)(ct1) ==\n hello)\n\n\n # Make sure all the results are the same\n okay = True\n r1 = results[0]\n for result in results[1:]:\n if r1[1] != result[1]:\n print('%s != %s' % (r1, result))\n okay = False\n assert(okay)\n\n # This verifies we can decrypt some snippets of data that were\n # generated with a previous iteration of mailpile.crypto.streamer\n from mailpile.util import sha512b64 as genkey\n legacy_data = \"part two, yeaaaah\\n\"\n legacy_nonce = \"2c1c43936034cae20eef86d961cb6570\"\n legacy_key = genkey(\"test key\", legacy_nonce)[:32].strip()\n legacy_ct = base64.b64decode(\"D+lBOPrtV+amUCAtoFPCzxsZ\")\n decrypted = aes_ctr_decrypt(legacy_key, legacy_nonce, legacy_ct)\n assert(legacy_data == decrypted)\n\n print(\"ok\")\n" }, { "path": "mailpile/crypto/autocrypt.py", "content": "from __future__ import print_function\n# Copyright (C) 2018 Jack Dodds & Mailpile ehf.\n# This code is part of Mailpile and is hereby released under the\n# Gnu Affero Public Licence v.3 - see ../../COPYING and ../../AGPLv3.txt.\n#\n\"\"\"\nThis file contains low-level Autocrypt code: constants, parsing, etc.\n\nThe higher level application logic (state database, persistance etc.) is\nmostly in mailpile.plugins.crypto_autocrypt, but inevitably some has\nleaked into mailpile.crypto.gpgi and mailpile.crypto.mime.\n\nExamples (and doctests):\n\n\n# Canonicalize e-mail addresses according to Autocrypt conventions\n>>> canonicalize_email('BrE@maILPile.is')\n'bre@mailpile.is'\n\n\n# Parse the Autocrypt header\n>>> ach = 'addr=bre@mailpile.is; _a=b; _c=d; keydata=aGVsbG8='\n>>> hv = parse_autocrypt_headervalue(ach, optional_attrs=['_a'])\n>>> hv['addr']\n'bre@mailpile.is'\n>>> hv['keydata']\n'hello'\n>>> hv['_a']\n'b'\n>>> hv.get('_c') is None\nTrue\n>>> hv.get('prefer-encrypt') is None\nTrue\n\n# Invalid autocrypt headers return {}\n>>> parse_autocrypt_headervalue('addr=bre@mailpile.is')\n{}\n>>> parse_autocrypt_headervalue('keydata=aGVsbG8=')\n{}\n>>> parse_autocrypt_headervalue('unknown=attribute; ' + ach)\n{}\n\n# Invalid prefer-encrypt values just get ignored\n>>> hv = parse_autocrypt_headervalue('prefer-encrypt=bogus; ' + ach)\n>>> hv.get('prefer-encrypt') is None\nTrue\n>>> hv = parse_autocrypt_headervalue('prefer-encrypt=mutual; ' + ach)\n>>> hv.get('prefer-encrypt') == 'mutual'\nTrue\n\n\n# Generate a valid Autocrypt header\n>>> make_autocrypt_header('bre@mailpile.is', 'hello', prefer_encrypt_mutual=True)\n'addr=bre@mailpile.is; prefer-encrypt=mutual; keydata=aGVsbG8='\n\n\n# Autocrypt setup-codes are used to secure our PGP keys\n>>> generate_autocrypt_setup_code(random_data='fake random garbage data')\n'1189-1868-6510-5211-5608-1629-1262-5635-4164'\n>>> len(generate_autocrypt_setup_code())\n44\n>>> generate_autocrypt_setup_code() != generate_autocrypt_setup_code()\nTrue\n\n\n# AutocryptRecommendations combine a key and a policy of what to do\n>>> ar = AutocryptRecommendation('disable')\n>>> ar.policy\n'disable'\n>>> ar.key_sig is None\nTrue\n\n# Combining recommendations for multiple parties has specific rules\n>>> ar2 = AutocryptRecommendation('encrypt', key_sig='12345')\n>>> AutocryptRecommendation.Synchronize(ar, ar2)\n'disable'\n>>> str(ar2)\n'disable'\n\n# Not just anything is a valid recommendation\n>>> ar2.policy = 'bogus'\nTraceback (most recent call last):\n ...\nValueError: Invalid Autocrypt policy: bogus\n\n\n\"\"\"\nimport base64\nimport datetime\nimport os\nimport pgpdump\nimport struct\nimport time\n\n\nAUTOCRYPT_IGNORE_MIMETYPES = ('multipart/report', )\n\n\ndef canonicalize_email(address):\n try:\n localpart, domain = address.split('@')\n except (ValueError, AttributeError):\n # Just return invalid e-mails unchanged, there is no sensible way\n # to canonicalize such a thing.\n return address\n\n # FIXME: Ensure domain is ASCII, if not, punycode it\n domain = domain.lower()\n\n # FIXME: Ensure we're using the \"empty locale\"\n localpart = localpart.lower()\n\n # NOTE: We deliberately do not strip plussed parts or perform any other\n # normalization of the localpart beyond lowercasing. This is both\n # to comply with the Autocrypt Level 1 spec, but also because being able\n # to use plussed parts to allow differing cryptographic identities to\n # share the same e-mail account is something power users like to do.\n\n return '%s@%s' % (localpart, domain)\n\n\ndef parse_autocrypt_headervalue(value, optional_attrs=None):\n # Based on:\n #\n # https://github.com/mailencrypt/inbome/blob/master/src/inbome/parse.py\n \"\"\"\n Parse an AutoCrypt header. Will return an empty dict if parsing fails.\n\n Optional attributes may be added to the result dictionary, but only the ones\n listed in optional_attrs (a list or dict); others are ignored.\n \"\"\"\n result_dict = {}\n try:\n for x in value.split(\";\"):\n kv = x.split(\"=\", 1)\n name = kv[0].strip()\n value = kv[1].strip()\n if name in (\"addr\", \"prefer-encrypt\"):\n result_dict[name] = value\n elif name == \"keydata\":\n keydata_base64 = \"\".join(value.split())\n keydata = base64.b64decode(keydata_base64)\n result_dict[name] = keydata\n elif name[:1] == '_':\n if optional_attrs and name in optional_attrs:\n result_dict[name] = value\n else:\n # Unknown value detected, refuse to parse any further\n return {}\n except (ValueError, TypeError, IndexError):\n return {}\n\n if \"keydata\" not in result_dict:\n # found no keydata, ignoring header\n return {}\n\n if \"addr\" not in result_dict:\n # found no e-mail address, ignoring header\n return {}\n else:\n result_dict[\"addr\"] = canonicalize_email(result_dict[\"addr\"])\n\n if result_dict.get(\"prefer-encrypt\") not in (\"mutual\", None):\n # Invalid prefer-encrypt value; treat as nopreference\n del result_dict['prefer-encrypt']\n\n return result_dict\n\n\ndef extract_autocrypt_header(msg, to=None, optional_attrs=None):\n # Autocrypt requires there only be one From header\n froms = msg.get_all(\"From\") or []\n if len(froms) != 1:\n return {}\n\n # Extract the from address for comparisons below. We compare the\n # canonicalized versions, which is not the strictest interpretation\n # of the spec, but feels like a reasonable balance here.\n from mailpile.mailutils.addresses import AddressHeaderParser\n from_addrs = AddressHeaderParser(froms[0])\n if len(from_addrs) != 1:\n return {}\n from_addr = canonicalize_email(from_addrs[0].address)\n\n to = canonicalize_email(to) if to else None\n all_results = []\n for inb in (msg.get_all(\"Autocrypt\") or []):\n res = parse_autocrypt_headervalue(inb, optional_attrs=optional_attrs)\n if res:\n if ((not to or canonicalize_email(res['addr']) == to) and\n (canonicalize_email(res['addr']) == from_addr)):\n all_results.append(res)\n\n # Return parsed header iff we found exactly one.\n if len(all_results) == 1:\n return all_results[0]\n else:\n return {}\n\n\ndef extract_autocrypt_gossip_headers(msg, to=None, optional_attrs=None):\n to = canonicalize_email(to) if to else None\n all_results = []\n for inb in (msg.get_all(\"Autocrypt-Gossip\") or []):\n res = parse_autocrypt_headervalue(inb, optional_attrs=optional_attrs)\n if res and (not to or res['addr'] == to):\n all_results.append(res)\n\n return all_results\n\n\ndef make_autocrypt_header(addr, binary_key,\n prefer_encrypt_mutual=False, prefix='Autocrypt'):\n prefix = '%s: ' % prefix\n pem = ' prefer-encrypt=mutual;' if prefer_encrypt_mutual else ''\n hdr = '%saddr=%s;%s keydata=' % (prefix, addr, pem)\n for c in base64.b64encode(binary_key).strip():\n if (len(hdr) % 78) == 0: hdr += ' '\n hdr += c\n return hdr[len(prefix):]\n\n\ndef generate_autocrypt_setup_code(random_data=None):\n \"\"\"\n Generate a passphrase/setup-code compliant with Autocrypt Level 1.\n\n From the spec: An Autocrypt Level 1 MUA MUST generate a Setup Code as\n UTF-8 string of 36 numeric characters, divided into nine blocks of four,\n separated by dashes. The dashes are part of the secret code and there\n are no spaces. This format holds about 119 bits of entropy. It is\n designed to be unambiguous, pronounceable, script-independent (Chinese,\n Cyrillic etc.), easily input on a mobile device and split into blocks\n that are easily kept in short term memory.\n \"\"\"\n random_data = random_data or os.urandom(16) # 16 bytes = 128 bits entropy\n ints = struct.unpack('>4I', random_data[:16])\n ival = ints[0] + (ints[1] << 32) + (ints[2] << 64) + (ints[3] << 96)\n blocks = []\n while len(blocks) < 9:\n blocks.append('%4.4d' % (ival % 10000))\n ival //= 10000\n return '-'.join(blocks)\n\n\n# FIXME: Add a with_signing_subkeys=True, implement. This deviates\n# from the Autocrypt spec, because Autocrypt says nothing about\n# signatures. But we're almost always signing our mail, and w/o\n# the subkeys the signatures cannot be checked.\ndef UNUSED_get_minimal_PGP_key(keydata,\n user_id=None, subkey_id=None, binary_out=False):\n \"\"\"\n Accepts a PGP key (armored or binary) and returns a minimal PGP key\n containing exactly five packets (base64 or binary) defining a\n primary key, a single user id with one self-signature, and a\n single encryption subkey with one self-signature. Such a five packet\n key MUST be used in Autocrypt headers (Level 1 Spec section 2.1.1).\n The unrevoked user id with newest unexpired self-signature and the\n unrevoked encryption-capable subkey with newest unexpired\n self-signature are selected from the input key.\n If user_id is provided, a user id containing that string will be\n selected if there is one, otherwise any user id will be accepted.\n If subkey_id is specified, only a subkey with that id will be selected.\n\n Along with the new key, the selected user id and subkey id are returned.\n Returns None if there is a failure.\n \"\"\"\n def _get_int4(data, offset):\n '''Pull four bytes from data at offset and return as an integer.'''\n return ((data[offset] << 24) + (data[offset + 1] << 16) +\n (data[offset + 2] << 8) + data[offset + 3])\n\n def _exp_time(creation_time, exp_time_subpacket_data):\n\n life_s = _get_int4(exp_time_subpacket_data, 0)\n if not life_s:\n return 0\n return packet.creation_time + datetime.timedelta( seconds = life_s)\n\n def _pgp_header(type, body_length):\n\n if body_length < 192:\n return bytearray([type+0xC0, body_length])\n elif body_length < 8384:\n return bytearray([type+0xC0, (body_length-192)//256+192,\n (body_length-192)%256])\n else:\n return bytearray([type+0xC0, 255,\n body_length//(1<<24), body_length//(1<<16) % 256,\n body_length//1<<8 % 256, body_length % 256])\n\n pri_key = None\n u_id = None\n u_id_sig = None\n u_id_match = False\n s_key = None\n s_key_sig = None\n user_id = canonicalize_email(user_id) if user_id else None\n now = datetime.datetime.utcfromtimestamp(time.time())\n\n if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in keydata:\n packet_iter = pgpdump.AsciiData(keydata).packets()\n else:\n packet_iter = pgpdump.BinaryData(keydata).packets()\n\n try:\n packet = next(packet_iter)\n except:\n packet = None\n\n while packet:\n\n if packet.raw == 6 and pri_key: # Primary key must be the first\n break # and only the first packet.\n elif packet.raw != 6 and not pri_key:\n break\n\n elif packet.raw == 6: # Primary Public-Key Packet\n pri_key = packet\n\n elif packet.raw == 13: # User ID Packet\n u_id_try = packet\n u_id_sig_try = None\n u_id_try_match = (\n not user_id or (user_id == canonicalize_email(u_id_try.user)))\n\n # Accept a nonmatching u_id IFF no other u_id matches.\n if u_id_match and not u_id_try_match:\n u_id_try = None\n\n for packet in packet_iter:\n if packet.raw != 2: # Signature Packet\n break\n elif not u_id_try:\n continue\n # User ID certification\n elif packet.raw_sig_type in (0x10, 0x11, 0x12, 0x13, 0x1F):\n if (pri_key.fingerprint.endswith(packet.key_id) and\n (not packet.expiration_time or\n packet.expiration_time > now) and\n (not u_id_sig_try or\n u_id_sig_try.creation_time\n < packet.creation_time)):\n u_id_sig_try = packet\n # Certification revocation\n elif packet.raw_sig_type == 0x30:\n if pri_key.fingerprint.endswith(packet.key_id):\n u_id_try = None\n u_id_sig_try = None\n\n # Select unrevoked user id with newest unexpired self-signature\n if u_id_try and u_id_sig_try and (\n not u_id or not u_id_sig or\n u_id_try_match and not u_id_match or\n u_id_sig_try.creation_time >= u_id_sig.creation_time):\n u_id = u_id_try\n u_id_sig = u_id_sig_try\n u_id_match = u_id_try_match\n continue # Skip next(packet_iter) - for has done it.\n\n elif packet.raw == 14: # Public-Subkey Packet\n s_key_try = packet\n s_key_sig_try = None\n\n # Honour a request for specific subkey and check for expiry.\n if ((subkey_id and not s_key_try.fingerprint.endswith(subkey_id))\n or (s_key_try.expiration_time and\n s_key_try.expiration_time < now)):\n s_key_try = None\n\n for packet in packet_iter:\n if packet.raw != 2: # Signature Packet\n break\n elif not s_key_try:\n continue\n # Subkey Binding Signature\n elif packet.raw_sig_type == 0x18:\n packet.key_expire_time = None\n if (pri_key.fingerprint.endswith(packet.key_id) and\n not packet.expiration_time or\n packet.expiration_time >= now):\n can_encrypt = True # Assume encrypt -- FIXME\n for subpacket in packet.subpackets:\n if subpacket.subtype == 9: # Key expiration\n packet.key_expire_time = _exp_time(\n packet.creation_time, subpacket.data)\n elif subpacket.subtype == 27: # Key flags\n can_encrypt |= subpacket.data[0] & 0x0C\n if can_encrypt and (not packet.key_expire_time or\n packet.key_expire_time >= now):\n s_key_sig_try = packet\n # Subkey revocation signature\n elif packet.raw_sig_type == 0x28:\n if pri_key.fingerprint.endswith(packet.key_id):\n s_key_try = None\n s_key_sig_try = None\n\n # Select unrevoked encryption-capable subkey with newest\n # unexpired self-signature (ignores newness of key itself).\n if s_key_try and s_key_sig_try and (not s_key_sig or\n s_key_sig_try.creation_time >= s_key_sig.creation_time):\n s_key = s_key_try\n s_key_sig = s_key_sig_try\n continue # Skip next(packet_iter) - for has done it.\n\n try:\n packet = next(packet_iter)\n except:\n packet = None\n\n if not(pri_key and u_id and u_id_sig and s_key and s_key_sig):\n return '', None, None\n\n newkey = (\n _pgp_header(pri_key.raw, len(pri_key.data)) + pri_key.data +\n _pgp_header(u_id.raw, len(u_id.data)) + u_id.data +\n _pgp_header(u_id_sig.raw, len(u_id_sig.data)) + u_id_sig.data +\n _pgp_header(s_key.raw, len(s_key.data)) + s_key.data +\n _pgp_header(s_key_sig.raw, len(s_key_sig.data)) + s_key_sig.data )\n\n if not binary_out:\n newkey = base64.b64encode(newkey)\n\n return newkey, u_id.user, s_key.key_id\n\n\nclass AutocryptRecommendation(object):\n DISABLE = \"disable\"\n DISCOURAGE = \"discourage\"\n ENABLE = \"enable\"\n ENCRYPT = \"encrypt\"\n\n ORDERED_POLICIES = (DISABLE, DISCOURAGE, ENABLE, ENCRYPT)\n\n def __init__(self, policy, key_sig=None):\n self.key_sig = self._policy = None\n self.set_recommendation(policy, key_sig)\n\n def __str__(self):\n if self.policy in (self.DISABLE,):\n return self.policy\n return \"%s (key=%s)\" % (self.policy, self.key_sig)\n\n @classmethod\n def Synchronize(cls, *recommendations):\n \"\"\"\n This will synchronize a set of Autocrypt recommendations to whatever\n the lowest common denomitor is, and then return that policy.\n \"\"\"\n if not recommendations:\n return cls.DISABLE\n lowest_common_policy = cls.ORDERED_POLICIES[min(\n cls.ORDERED_POLICIES.index(r.policy) for r in recommendations)]\n for r in recommendations:\n r.policy = lowest_common_policy\n return lowest_common_policy\n\n def set_recommendation(self, policy, key_sig=None):\n if policy not in self.ORDERED_POLICIES:\n raise ValueError('Invalid Autocrypt policy: %s' % policy)\n if policy != self.DISABLE and key_sig is None and self.key_sig is None:\n raise ValueError('Policy %s requires a key' % policy)\n self._policy = policy\n if key_sig is not None:\n self.key_sig = key_sig\n\n policy = property(lambda self: self._policy, set_recommendation)\n\n\nif __name__ == \"__main__\":\n import sys\n import doctest\n\n results = doctest.testmod(optionflags=doctest.ELLIPSIS)\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/crypto/gpgi.py", "content": "#coding:utf-8\nfrom __future__ import print_function\nimport os\nimport string\nimport sys\nimport time\nimport re\nimport StringIO\nimport tempfile\nimport threading\nimport traceback\nimport urllib\nimport select\nimport pgpdump\nimport pgpdump.utils\nimport base64\nimport quopri\nfrom datetime import datetime\nfrom email.parser import Parser\nfrom email.message import Message\nfrom threading import Thread\n\nimport mailpile.platforms\nfrom mailpile.i18n import gettext\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.crypto.state import *\nfrom mailpile.crypto.mime import MimeSigningWrapper, MimeEncryptingWrapper\nfrom mailpile.safe_popen import Popen, PIPE, Safe_Pipe\n\n\n_ = lambda s: s\n\nDEFAULT_KEYSERVERS = [\"hkps://hkps.pool.sks-keyservers.net\",\n \"hkp://subset.pool.sks-keyservers.net\"]\nDEFAULT_KEYSERVER_OPTIONS = [\n 'ca-cert-file=%s' % __file__.replace('.pyc', '.py')]\nDEFAULT_IMPORT_OPTIONS = [\n 'import-minimal']\n\nGPG_KEYID_LENGTH = 8\nGNUPG_HOMEDIR = None # None=use what gpg uses\nGPG_BINARY = mailpile.platforms.GetDefaultGnuPGCommand\nGPG_VERSIONS = {}\nBLOCKSIZE = 65536\n\nopenpgp_algorithms = {1: _(\"RSA\"),\n 2: _(\"RSA (encrypt only)\"),\n 3: _(\"RSA (sign only)\"),\n 16: _(\"ElGamal (encrypt only)\"),\n 17: _(\"DSA\"),\n 20: _(\"ElGamal (encrypt/sign) [COMPROMISED]\"),\n 22: _(\"EdDSA\"),\n 999: _(\"Unknown\")}\n# For details on type 20 compromisation, see\n# http://lists.gnupg.org/pipermail/gnupg-announce/2003q4/000160.html\n\nENTROPY_LOCK = threading.Lock()\n\nclass GnuPGEventUpdater:\n \"\"\"\n Parse the GPG response into something useful for the Event Log.\n \"\"\"\n def __init__(self, event):\n from mailpile.eventlog import Event\n self.event = event or Event()\n\n def _log(self, section, message):\n data = section.get('gnupg', [])\n if data:\n data[-1].append(message)\n\n def _log_private(self, message):\n self._log(self.event.private_data, message)\n\n def _log_public(self, message):\n self._log(self.event.private_data, message)\n self._log(self.event.data, message)\n\n def running_gpg(self, why):\n for section in (self.event.data, self.event.private_data):\n data = section.get('gnupg', [])\n data.append([why, int(time.time())])\n section['gnupg'] = data\n\n def update_args(self, args):\n self._log_public(' '.join(args))\n\n def update_sent_passphrase(self):\n self._log_public('Sent passphrase')\n\n def _parse_gpg_line(self, line):\n if line.startswith('[GNUPG:] '):\n pass # FIXME: Parse for machine-readable data\n elif line.startswith('gpg: '):\n self._log_private(line[5:].strip())\n\n def update_stdout(self, line):\n self._parse_gpg_line(line)\n\n def update_stderr(self, line):\n self._parse_gpg_line(line)\n\n def update_return_code(self, code):\n self._log_public('GnuPG returned %s' % code)\n\n\nclass GnuPGResultParser:\n \"\"\"\n Parse the GPG response into EncryptionInfo and SignatureInfo.\n \"\"\"\n def __init__(rp, decrypt_requires_MDC=True, debug=None):\n rp.decrypt_requires_MDC = decrypt_requires_MDC\n rp.debug = debug or (lambda t: True)\n\n rp.signature_info = SignatureInfo()\n rp.signature_info[\"protocol\"] = \"openpgp\"\n\n rp.encryption_info = EncryptionInfo()\n rp.encryption_info[\"protocol\"] = \"openpgp\"\n\n rp.plaintext = \"\"\n\n def parse(rp, retvals):\n signature_info = rp.signature_info\n encryption_info = rp.encryption_info\n from mailpile.mailutils.emails import ExtractEmailAndName\n\n # Belt & suspenders: work around some buggy GnuPG status codes\n gpg_stderr = ''.join(retvals[1][\"stderr\"])\n\n # First pass, set some initial state.\n locked, missing = [], []\n for data in retvals[1][\"status\"]:\n keyword = data[0].strip() # The last keyword often ends in \\n\n\n if keyword == 'NEED_PASSPHRASE':\n locked += [data[2]]\n encryption_info.part_status = \"lockedkey\"\n encryption_info[\"locked_keys\"] = list(set(locked))\n\n elif keyword == 'GOOD_PASSPHRASE':\n encryption_info[\"locked_keys\"] = []\n\n elif keyword == \"DECRYPTION_FAILED\":\n missing += [x[1].strip() for x in retvals[1][\"status\"]\n if x[0] == \"NO_SECKEY\"]\n if missing:\n encryption_info[\"missing_keys\"] = list(set(missing))\n if encryption_info.part_status != \"lockedkey\":\n if missing:\n encryption_info.part_status = \"missingkey\"\n else:\n encryption_info.part_status = \"error\"\n\n elif keyword == \"DECRYPTION_OKAY\":\n if (rp.decrypt_requires_MDC and\n 'message was not integrity protected' in gpg_stderr):\n rp.debug('Message not integrity protected, failing.')\n encryption_info.part_status = \"error\"\n else:\n encryption_info.part_status = \"decrypted\"\n rp.plaintext = \"\".join(retvals[1][\"stdout\"])\n\n elif keyword == \"ENC_TO\":\n keylist = encryption_info.get(\"have_keys\", [])\n if data[1] not in keylist:\n keylist.append(data[1].strip())\n encryption_info[\"have_keys\"] = list(set(keylist))\n\n elif keyword == \"PLAINTEXT\":\n encryption_info.filename = data[3].strip()\n\n elif signature_info.part_status == \"none\":\n # Only one of these will ever be emitted per key, use\n # this to set initial state. We may end up revising\n # the status depending on more info later.\n if keyword in (\"GOODSIG\", \"BADSIG\"):\n email, fn = ExtractEmailAndName(\n \" \".join(data[2:]).decode('utf-8'))\n signature_info[\"name\"] = fn\n signature_info[\"email\"] = email\n signature_info.part_status = ((keyword == \"GOODSIG\")\n and \"unverified\"\n or \"invalid\")\n rp.plaintext = \"\".join(retvals[1][\"stdout\"])\n\n elif keyword == \"ERRSIG\":\n signature_info.part_status = \"error\"\n signature_info[\"keyinfo\"] = data[1]\n signature_info[\"timestamp\"] = int(data[5])\n\n # Second pass, this may update/mutate the state set above\n for data in retvals[1][\"status\"]:\n keyword = data[0].strip() # The last keyword often ends in \\n\n\n if keyword == \"NO_SECKEY\":\n keyid = data[1].strip()\n if \"missing_keys\" not in encryption_info:\n encryption_info[\"missing_keys\"] = [keyid]\n elif keyid not in encryption_info[\"missing_keys\"]:\n encryption_info[\"missing_keys\"].append(keyid)\n while keyid in encryption_info[\"have_keys\"]:\n encryption_info[\"have_keys\"].remove(keyid)\n\n elif keyword == \"VALIDSIG\":\n # FIXME: Determine trust level, between new, unverified,\n # verified, untrusted.\n signature_info[\"keyinfo\"] = data[1]\n signature_info[\"timestamp\"] = int(data[3])\n\n elif keyword in (\"EXPKEYSIG\", \"REVKEYSIG\"):\n email, fn = ExtractEmailAndName(\n \" \".join(data[2:]).decode('utf-8'))\n signature_info[\"name\"] = fn\n signature_info[\"email\"] = email\n signature_info.part_status = ((keyword == \"EXPKEYSIG\")\n and \"expired\"\n or \"revoked\")\n\n # FIXME: This appears to be spammy. Is my key borked, or\n # is GnuPG being stupid?\n #\n # elif keyword == \"KEYEXPIRED\": # Ignoring: SIGEXPIRED\n # signature_info.part_status = \"expired\"\n elif keyword == \"KEYREVOKED\":\n signature_info.part_status = \"revoked\"\n elif keyword == \"NO_PUBKEY\":\n signature_info.part_status = \"unknown\"\n\n elif keyword in (\"TRUST_ULTIMATE\", \"TRUST_FULLY\"):\n if signature_info.part_status == \"unverified\":\n signature_info.part_status = \"verified\"\n elif (keyword == \"DECRYPTION_INFO\" and\n encryption_info.part_status == \"decrypted\"\n and rp.decrypt_requires_MDC):\n mdc_method = data[1].strip()\n aead_algo = data[3].strip() if len(data) > 3 else 0\n if not mdc_method and not aeadalgo:\n encryption_info.part_status = \"error\"\n\n if encryption_info.part_status == \"error\":\n rp.plaintext = \"\"\n\n return rp\n\n\nclass GnuPGRecordParser:\n def __init__(self):\n self.keys = {}\n self.curkeyid = None\n self.curdata = None\n\n self.record_fields = [\"record\", \"validity\", \"keysize\", \"keytype\",\n \"keyid\", \"creation_date\", \"expiration_date\",\n \"uidhash\", \"ownertrust\", \"uid\", \"sigclass\",\n \"capabilities\", \"flag\", \"sn\", \"hashtype\",\n \"curve\"]\n self.record_types = [\"pub\", \"sub\", \"ssb\", \"fpr\", \"uat\", \"sec\", \"tru\",\n \"sig\", \"rev\", \"uid\", \"gpg\", \"rvk\", \"grp\"]\n self.record_parsers = [self.parse_pubkey, self.parse_subkey,\n self.parse_subkey, self.parse_fingerprint,\n self.parse_userattribute, self.parse_privkey,\n self.parse_trust, self.parse_signature,\n self.parse_revoke, self.parse_uidline,\n self.parse_none, self.parse_revocation_key,\n self.parse_keygrip]\n\n self.dispatch = dict(zip(self.record_types, self.record_parsers))\n\n def parse(self, lines):\n for line in lines:\n self.parse_line(line)\n return self.keys\n\n def parse_line(self, line):\n line = dict(zip(self.record_fields,\n map(lambda s: s.replace(\"\\\\x3a\", \":\"),\n stubborn_decode(line).strip().split(\":\"))))\n r = self.dispatch.get(line[\"record\"], self.parse_unknown)\n r(line)\n\n def _parse_dates(self, line):\n for ts in ('expiration_date', 'creation_date'):\n if line.get(ts) and '-' not in line[ts]:\n line[ts+'_ts'] = line[ts]\n try:\n unixtime = int(line[ts])\n if unixtime > 946684800: # 2000-01-01\n dt = datetime.fromtimestamp(unixtime)\n line[ts] = dt.strftime('%Y-%m-%d')\n except ValueError:\n line[ts] = '1970-01-01'\n\n def _parse_keydata(self, line):\n line[\"keytype_name\"] = _(openpgp_algorithms.get(int(line[\"keytype\"]),\n 'Unknown'))\n line[\"capabilities_map\"] = {\n \"encrypt\": \"E\" in line[\"capabilities\"],\n \"sign\": \"S\" in line[\"capabilities\"],\n \"certify\": \"C\" in line[\"capabilities\"],\n \"authenticate\": \"A\" in line[\"capabilities\"],\n }\n line[\"disabled\"] = \"D\" in line[\"capabilities\"]\n line[\"revoked\"] = \"r\" in line[\"validity\"]\n line[\"expired\"] = \"e\" in line[\"validity\"]\n\n self._parse_dates(line)\n\n return line\n\n def _clean_curdata(self):\n for v in self.curdata.keys():\n if self.curdata[v] == \"\":\n del self.curdata[v]\n del self.curdata[\"record\"]\n\n def parse_pubkey(self, line):\n self.curkeyid = line[\"keyid\"]\n self.curdata = self.keys[self.curkeyid] = self._parse_keydata(line)\n self.curdata[\"subkeys\"] = []\n self.curdata[\"uids\"] = []\n self.curdata[\"secret\"] = (self.curdata[\"record\"] == \"sec\")\n self.parse_uidline(self.curdata)\n self._clean_curdata()\n\n def parse_subkey(self, line):\n self.curdata = self._parse_keydata(line)\n self.keys[self.curkeyid][\"subkeys\"].append(self.curdata)\n self._clean_curdata()\n\n def parse_fingerprint(self, line):\n fpr = line[\"uid\"]\n self.curdata[\"fingerprint\"] = fpr\n if len(self.curkeyid) < len(fpr):\n self.keys[fpr] = self.keys[self.curkeyid]\n del(self.keys[self.curkeyid])\n self.curkeyid = fpr\n\n def parse_userattribute(self, line):\n # TODO: We are currently ignoring user attributes as not useful.\n # We may at some point want to use --attribute-fd and read\n # in user photos and such?\n pass\n\n def parse_privkey(self, line):\n self.parse_pubkey(line)\n\n def parse_uidline(self, line):\n email, name, comment = parse_uid(line[\"uid\"])\n self._parse_dates(line)\n if email or name or comment:\n self.keys[self.curkeyid][\"uids\"].append({\n \"email\": email,\n \"name\": name,\n \"comment\": comment,\n \"creation_date\": line[\"creation_date\"]\n })\n else:\n pass # This is the case where a uid or sec line have no\n # information aside from the creation date, which we\n # parse elsewhere. As these lines are effectively blank,\n # we omit them to simplify presentation to the user.\n\n def parse_trust(self, line):\n # FIXME: We are currently ignoring commentary from the Trust DB.\n pass\n\n def parse_signature(self, line):\n # FIXME: This is probably wrong; signatures are on UIDs and not\n # the key itself. No? Yes? Figure this out.\n if \"signatures\" not in self.keys[self.curkeyid]:\n self.keys[self.curkeyid][\"signatures\"] = []\n sig = {\n \"signer\": line[9],\n \"signature_date\": line[5],\n \"keyid\": line[4],\n \"trust\": line[10],\n \"keytype\": line[4]\n }\n self.keys[self.curkeyid][\"signatures\"].append(sig)\n\n def parse_keygrip(self, line):\n self.curdata[\"keygrip\"] = line[\"uid\"]\n\n def parse_revoke(self, line):\n pass # FIXME\n\n def parse_revocation_key(self, line):\n pass # FIXME\n\n def parse_unknown(self, line):\n print(\"Unknown line with code '%s'\" % (line,))\n\n def parse_none(line):\n pass\n\n\nUID_PARSE_RE = \"^([^\\(\\<]+?){0,1}( \\((.+?)\\)){0,1}( \\<(.+?)\\>){0,1}\\s*$\"\n\n\ndef stubborn_decode(text):\n if isinstance(text, unicode):\n return text\n try:\n return text.decode(\"utf-8\")\n except UnicodeDecodeError:\n try:\n return text.decode(\"iso-8859-1\")\n except UnicodeDecodeError:\n return uidstr.decode(\"utf-8\", \"replace\")\n\n\ndef parse_uid(uidstr):\n matches = re.match(UID_PARSE_RE, uidstr)\n if matches:\n email = matches.groups(0)[4] or \"\"\n comment = matches.groups(0)[2] or \"\"\n name = matches.groups(0)[0] or \"\"\n else:\n if '@' in uidstr and ' ' not in uidstr:\n email, name = uidstr, \"\"\n else:\n email, name = \"\", uidstr\n comment = \"\"\n\n return email, name, comment\n\n\nclass StreamReader(Thread):\n def __init__(self, name, fd, callback, lines=True):\n Thread.__init__(self, target=self.readin, args=(fd, callback))\n self.name = name\n self.state = 'startup'\n self.lines = lines\n self.start()\n\n def __str__(self):\n return '%s(%s/%s, lines=%s)' % (Thread.__str__(self),\n self.name, self.state, self.lines)\n\n def readin(self, fd, callback):\n try:\n if self.lines:\n self.state = 'read'\n for line in iter(fd.readline, b''):\n self.state = 'callback'\n callback(line)\n self.state = 'read'\n else:\n while True:\n self.state = 'read'\n buf = fd.read(BLOCKSIZE)\n self.state = 'callback'\n callback(buf)\n if buf == \"\":\n break\n except:\n traceback.print_exc()\n finally:\n self.state = 'done'\n fd.close()\n\n\nclass StreamWriter(Thread):\n def __init__(self, name, fd, output, partial_write_ok=False):\n Thread.__init__(self, target=self.writeout, args=(fd, output))\n self.name = name\n self.state = 'startup'\n self.partial_write_ok = partial_write_ok\n self.start()\n\n def __str__(self):\n return '%s(%s/%s)' % (Thread.__str__(self), self.name, self.state)\n\n def writeout(self, fd, output):\n if isinstance(output, (str, unicode, bytearray)):\n total = len(output)\n output = StringIO.StringIO(output)\n else:\n total = 0\n try:\n while True:\n self.state = 'read'\n line = output.read(BLOCKSIZE)\n if line == \"\":\n break\n self.state = 'write'\n fd.write(line)\n total -= len(line)\n output.close()\n except:\n if not self.partial_write_ok:\n print('%s: %s bytes left' % (self, total))\n traceback.print_exc()\n finally:\n self.state = 'done'\n fd.close()\n\n\nDEBUG_GNUPG = False\n\nclass GnuPG:\n \"\"\"\n Wrap GnuPG and make all functionality feel Pythonic.\n \"\"\"\n ARMOR_BEGIN_SIGNED = '-----BEGIN PGP SIGNED MESSAGE-----'\n ARMOR_BEGIN_SIGNATURE = '-----BEGIN PGP SIGNATURE-----'\n ARMOR_END_SIGNED = '-----END PGP SIGNATURE-----'\n ARMOR_END_SIGNATURE = '-----END PGP SIGNATURE-----'\n\n ARMOR_BEGIN_ENCRYPTED = '-----BEGIN PGP MESSAGE-----'\n ARMOR_END_ENCRYPTED = '-----END PGP MESSAGE-----'\n\n ARMOR_BEGIN_PUB_KEY = '-----BEGIN PGP PUBLIC KEY BLOCK-----'\n ARMOR_END_PUB_KEY = '-----END PGP PUBLIC KEY BLOCK-----'\n\n LAST_KEY_USED = 'DEFAULT' # This is a 1-value global cache\n\n def __init__(self, config,\n session=None, use_agent=None, debug=False, dry_run=False,\n event=None, passphrase=None):\n global DEBUG_GNUPG\n self.available = None\n self.outputfds = [\"stdout\", \"stderr\", \"status\"]\n self.errors = []\n self.event = GnuPGEventUpdater(event)\n self.session = session\n self.config = config or (session and session.config) or None\n self.status_filenames = []\n if self.config:\n DEBUG_GNUPG = ('gnupg' in self.config.sys.debug)\n self.homedir = self.config.sys.gpg_home or GNUPG_HOMEDIR\n self.gpgbinary = self.config.sys.gpg_binary or GPG_BINARY()\n self.passphrases = self.config.passphrases\n self.passphrase = (passphrase if (passphrase is not None) else\n self.passphrases['DEFAULT']).get_reader()\n self.use_agent = (use_agent if (use_agent is not None)\n else self.config.prefs.gpg_use_agent)\n else:\n self.homedir = GNUPG_HOMEDIR\n self.gpgbinary = GPG_BINARY()\n self.passphrases = None\n if passphrase:\n self.passphrase = passphrase.get_reader()\n else:\n self.passphrase = None\n self.use_agent = use_agent\n self.dry_run = dry_run\n self.debug = (self._debug_all if (debug or DEBUG_GNUPG)\n else self._debug_none)\n\n def prepare_passphrase(self, keyid, signing=False, decrypting=False):\n \"\"\"Query the Mailpile secrets for a usable passphrase.\"\"\"\n def _use(kid, sps_reader):\n self.passphrase = sps_reader\n GnuPG.LAST_KEY_USED = kid\n return True\n\n if self.config:\n message = []\n if decrypting:\n message.append(_(\"Your PGP key is needed for decrypting.\"))\n if signing:\n message.append(_(\"Your PGP key is needed for signing.\"))\n match, sps = self.config.get_passphrase(keyid,\n prompt=_('Unlock your encryption key'),\n description=' '.join(message))\n if match:\n return _use(match, sps.get_reader())\n\n self.passphrase = None # This *may* allow use of the GnuPG agent\n return False\n\n def _debug_all(self, msg):\n if self.session:\n self.session.debug(msg.rstrip())\n else:\n print('%s' % str(msg).rstrip())\n\n def _debug_none(self, msg):\n pass\n\n def set_home(self, path):\n self.homedir = path\n\n def version(self):\n \"\"\"Returns a string representing the GnuPG version number.\"\"\"\n self.event.running_gpg(_('Checking GnuPG version'))\n retvals = self.run([\"--version\"], novercheck=True)\n return retvals[1][\"stdout\"][0].split('\\n')[0]\n\n def version_tuple(self, update=False):\n \"\"\"Returns a tuple representing the GnuPG version number.\"\"\"\n global GPG_VERSIONS\n if update or not GPG_VERSIONS.get(self.gpgbinary):\n match = re.search( \"(\\d+).(\\d+).(\\d+)\", self.version() )\n version = tuple(int(v) for v in match.groups())\n GPG_VERSIONS[self.gpgbinary] = version\n return GPG_VERSIONS[self.gpgbinary]\n\n def gnupghome(self):\n \"\"\"Returns the location of the GnuPG keyring\"\"\"\n self.event.running_gpg(_('Checking GnuPG home directory'))\n rv = self.run([\"--version\"], novercheck=True)[1][\"stdout\"][0]\n for l in rv.splitlines():\n if l.startswith('Home: '):\n return os.path.expanduser(l[6:].strip())\n return os.path.expanduser(os.getenv('GNUPGHOME', '~/.gnupg'))\n\n def is_available(self):\n try:\n self.event.running_gpg(_('Checking GnuPG availability'))\n self.version_tuple(update=True)\n self.available = True\n except OSError:\n self.available = False\n\n return self.available\n\n def common_args(self,\n args=None, version=None,\n will_send_passphrase=False,\n interactive=False):\n if args is None:\n args = []\n if version is None:\n version = self.version_tuple()\n\n args.insert(0, self.gpgbinary)\n args.insert(1, \"--utf8-strings\")\n\n # Disable SHA1 and compression in all things GnuPG\n args[1:1] = [\"--personal-digest-preferences=SHA512\",\n \"--personal-compress-preferences=Uncompressed\",\n \"--digest-algo=SHA512\",\n \"--cert-digest-algo=SHA512\"]\n\n if self.homedir:\n args.insert(1, \"--homedir=%s\" % self.homedir)\n\n if version > (2, 1, 11):\n binaries = mailpile.platforms.DetectBinaries()\n for which, setting in (('GnuPG_dirmngr', 'dirmngr-program'),\n ('GnuPG_agent', 'agent-program')):\n if which in binaries:\n args.insert(1, \"--%s=%s\" % (setting, binaries[which]))\n else:\n print('wtf: %s not in %s' % (which, binaries))\n\n if (not self.use_agent) or will_send_passphrase:\n if version < (1, 5):\n args.insert(1, \"--no-use-agent\")\n elif version > (2, 1, 11):\n args.insert(1, \"--pinentry-mode=loopback\")\n else:\n raise ImportError('Mailpile requires GnuPG 1.4.x or 2.1.12+ !')\n\n if not interactive:\n args.insert(1, \"--with-colons\")\n args.insert(1, \"--verbose\")\n args.insert(1, \"--batch\")\n args.insert(1, \"--enable-progress-filter\")\n if self.status_filenames:\n args.insert(1, \"--status-file=%s\" % self.status_filenames[-1])\n if will_send_passphrase:\n args.insert(2, \"--passphrase-fd=0\")\n\n if self.dry_run:\n args.insert(1, \"--dry-run\")\n\n return args\n\n def run(self, *args, **kwargs):\n # This wrapper handles temporary status files. Since we may recursively\n # invoke ourselves, we keep a stack of tempfiles and push/pop from the\n # list.\n fd = tempfile.NamedTemporaryFile(delete=False)\n fd.close() # Avoid potential conflicts on Windows\n try:\n self.status_filenames.append(fd.name)\n return self.run_without_status(*args, **kwargs)\n finally:\n os.remove(self.status_filenames.pop(-1))\n\n def run_without_status(self,\n args=None, gpg_input=None, outputfd=None, partial_read_ok=False,\n send_passphrase=False, _raise=None, novercheck=False):\n if novercheck:\n version = (1, 4)\n else:\n version = self.version_tuple()\n\n args = self.common_args(\n args=list(args if args else []),\n version=version,\n will_send_passphrase=(self.passphrase and send_passphrase))\n\n self.outputbuffers = dict([(x, []) for x in self.outputfds])\n self.threads = {}\n gpg_retcode = -1\n proc = None\n try:\n if send_passphrase and (self.passphrase is None):\n self.debug('Running WITHOUT PASSPHRASE %s' % ' '.join(args))\n self.debug(''.join(traceback.format_stack()))\n else:\n self.debug('Running %s' % ' '.join(args))\n\n # Here we go!\n self.event.update_args(args)\n proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=0)\n\n # GnuPG is a bit crazy, and requires that the passphrase\n # be sent and the filehandle closed before anything else\n # interesting happens.\n if send_passphrase and self.passphrase is not None:\n self.passphrase.seek(0, 0)\n c = self.passphrase.read(BLOCKSIZE)\n while c != '':\n proc.stdin.write(c)\n c = self.passphrase.read(BLOCKSIZE)\n proc.stdin.write('\\n')\n self.event.update_sent_passphrase()\n\n wtf = ' '.join(args)\n self.threads = {\n \"stderr\": StreamReader('gpgi-stderr(%s)' % wtf,\n proc.stderr, self.parse_stderr)\n }\n\n if outputfd:\n self.threads[\"stdout\"] = StreamReader(\n 'gpgi-stdout-to-fd(%s)' % wtf,\n proc.stdout, outputfd.write, lines=False)\n else:\n self.threads[\"stdout\"] = StreamReader(\n 'gpgi-stdout-parsed(%s)' % wtf,\n proc.stdout, self.parse_stdout)\n\n if gpg_input:\n # If we have output, we just stream it. Technically, this\n # doesn't really need to be a thread at the moment.\n self.debug('< 1:\n print('WARNING: Failed to reap thread %s' % thr)\n\n def parse_status(self, line, *args):\n self.debug('<>> g = GnuPG(None)\n >>> g.list_keys()[0]\n 0\n \"\"\"\n list_keys = [\"--fingerprint\"]\n for sel in set(selectors or []):\n list_keys += [\"--list-keys\", sel]\n if not selectors:\n list_keys += [\"--list-keys\"]\n self.event.running_gpg(_('Fetching GnuPG public key list (selectors=%s)'\n ) % ', '.join(selectors or []))\n retvals = self.run(list_keys)\n return self.parse_keylist(retvals[1][\"stdout\"])\n\n def list_secret_keys(self, selectors=None):\n #\n # Note: The selectors that are passed by default work around a bug\n # in GnuPG < 2.1, where --list-secret-keys does not list\n # details about key capabilities or expiry for\n # --list-secret-keys unless a selector is provided. A dot\n # is reasonably likely to appear in all PGP keys, as it is\n # a common component of e-mail addresses (and @ does not\n # work as a selector for some reason...)\n #\n # The downside of this workaround is that keys with no e-mail\n # address or an address like alice@localhost won't be found.\n # So we disable this hack on GnuPG >= 2.1.\n #\n if not selectors and self.version_tuple() < (2, 1):\n selectors = [\".\", \"a\", \"e\", \"i\", \"p\", \"t\", \"k\"]\n\n list_keys = [\"--fingerprint\"]\n if selectors:\n for sel in selectors:\n # FIXME - In 2.1.18 and 1.4.21 only one --list-keys is needed.\n list_keys += [\"--list-secret-keys\", sel]\n else:\n list_keys += [\"--list-secret-keys\"]\n\n self.event.running_gpg(_('Fetching GnuPG secret key list (selectors=%s)'\n ) % ', '.join(selectors or ['None']))\n retvals = self.run(list_keys)\n secret_keys = self.parse_keylist(retvals[1][\"stdout\"])\n\n # Another unfortunate thing GnuPG < 2.1 does, is it hides the disabled\n # state when listing secret keys; it seems internally only the\n # public key is disabled. This makes it hard for us to reason about\n # which keys can actually be used, so we compensate...\n list_keys = [\"--fingerprint\"]\n for fprint in set(secret_keys):\n # FIXME - In both 2.1.18 and 1.4.21 only one --list-keys is needed.\n list_keys += [\"--list-keys\", fprint]\n retvals = self.run(list_keys)\n public_keys = self.parse_keylist(retvals[1][\"stdout\"])\n for fprint, info in public_keys.iteritems():\n if fprint in set(secret_keys):\n for k in (\"disabled\", \"revoked\", \"expired\"):\n secret_keys[fprint][k] = info[k]\n\n return secret_keys\n\n def import_keys(self,\n key_data=None,\n import_options=DEFAULT_IMPORT_OPTIONS,\n filter_uid_emails=None):\n \"\"\"\n Imports gpg keys from a file object or string.\n >>> key_data = open(\"testing/pub.key\").read()\n >>> g = GnuPG(None)\n >>> g.import_keys(key_data)\n {'failed': [], 'updated': [{'details_text': 'unchanged', 'details': 0, 'fingerprint': '08A650B8E2CBC1B02297915DC65626EED13C70DA'}], 'imported': [], 'results': {'sec_dups': 0, 'unchanged': 1, 'num_uids': 0, 'skipped_new_keys': 0, 'no_userids': 0, 'num_signatures': 0, 'num_revoked': 0, 'sec_imported': 0, 'sec_read': 0, 'not_imported': 0, 'count': 1, 'imported_rsa': 0, 'imported': 0, 'num_subkeys': 0}}\n \"\"\"\n self.event.running_gpg(_('Importing key to GnuPG key chain'))\n cmd = [\"--import\"]\n if filter_uid_emails:\n expr = ' || '.join('uid =~ %s' % e for e in filter_uid_emails)\n cmd[1:1] = ['--import-filter', 'keep-uid=%s' % expr]\n for opt in import_options:\n cmd[1:1] = ['--import-options', opt]\n retvals = self.run(cmd, gpg_input=key_data)\n return self._parse_import(retvals[1][\"status\"])\n\n def _parse_import(self, output):\n res = {\"imported\": [], \"updated\": [], \"failed\": []}\n for x in output:\n if x[0] == \"IMPORTED\":\n res[\"imported\"].append({\n \"fingerprint\": x[1],\n \"username\": x[2].rstrip()\n })\n elif x[0] == \"IMPORT_OK\":\n reasons = {\n \"0\": \"unchanged\",\n \"1\": \"new key\",\n \"2\": \"new user IDs\",\n \"4\": \"new signatures\",\n \"8\": \"new subkeys\",\n \"16\": \"contains private key\",\n \"17\": \"contains new private key\",\n }\n res[\"updated\"].append({\n \"details\": int(x[1]),\n # FIXME: Reasons may be ORed! This does NOT handle that.\n \"details_text\": reasons.get(x[1], str(x[1])),\n \"fingerprint\": x[2].rstrip(),\n })\n elif x[0] == \"IMPORT_PROBLEM\":\n reasons = {\n \"0\": \"no reason given\",\n \"1\": \"invalid certificate\",\n \"2\": \"issuer certificate missing\",\n \"3\": \"certificate chain too long\",\n \"4\": \"error storing certificate\",\n }\n res[\"failed\"].append({\n \"details\": int(x[1]),\n \"details_text\": reasons.get(x[1], str(x[1])),\n \"fingerprint\": x[2].rstrip()\n })\n elif x[0] == \"IMPORT_RES\":\n res[\"results\"] = {\n \"count\": int(x[1]),\n \"no_userids\": int(x[2]),\n \"imported\": int(x[3]),\n \"imported_rsa\": int(x[4]),\n \"unchanged\": int(x[5]),\n \"num_uids\": int(x[6]),\n \"num_subkeys\": int(x[7]),\n \"num_signatures\": int(x[8]),\n \"num_revoked\": int(x[9]),\n \"sec_read\": int(x[10]),\n \"sec_imported\": int(x[11]),\n \"sec_dups\": int(x[12]),\n \"skipped_new_keys\": int(x[13]),\n \"not_imported\": int(x[14].rstrip()),\n }\n return res\n\n def decrypt(self, data,\n outputfd=None, passphrase=None, as_lines=False, require_MDC=True):\n \"\"\"\n Note that this test will fail if you don't replace the recipient with\n one whose key you control.\n >>> g = GnuPG(None)\n >>> ct = g.encrypt(\"Hello, World\", to=[\"smari@mailpile.is\"])[1]\n >>> g.decrypt(ct)[\"text\"]\n 'Hello, World'\n \"\"\"\n if passphrase is not None:\n self.passphrase = passphrase.get_reader()\n elif GnuPG.LAST_KEY_USED:\n # This is an opportunistic approach to passphrase usage... we\n # just hope the passphrase we used last time will work again.\n # If we are right, we are done. If we are wrong, the output\n # will tell us which key IDs to look for in our secret stash.\n self.prepare_passphrase(GnuPG.LAST_KEY_USED, decrypting=True)\n\n self.event.running_gpg(_('Decrypting %d bytes of data') % len(data))\n for tries in (1, 2):\n retvals = self.run([\"--decrypt\"], gpg_input=data,\n outputfd=outputfd,\n send_passphrase=True)\n if tries == 1:\n keyid = None\n for msg in reversed(retvals[1]['status']):\n # Reverse order so DECRYPTION_OKAY overrides KEY_CONSIDERED.\n # If decryption is not ok, look for good passphrase, retry.\n if msg[0] == 'DECRYPTION_OKAY':\n break\n elif (msg[0] == 'NEED_PASSPHRASE') and (passphrase is None):\n # This message is output by gpg 1.4 but not 2.1.\n if self.prepare_passphrase(msg[2], decrypting=True):\n keyid = msg[2]\n break\n elif (msg[0] == 'KEY_CONSIDERED') and (passphrase is None):\n # This message is output by gpg 2.1 but not 1.4.\n if self.prepare_passphrase(msg[1], decrypting=True):\n keyid = msg[1]\n break\n if not keyid:\n break\n\n if as_lines:\n as_lines = retvals[1][\"stdout\"]\n retvals[1][\"stdout\"] = []\n\n rp = GnuPGResultParser(decrypt_requires_MDC=require_MDC,\n debug=self.debug).parse(retvals)\n return (rp.signature_info, rp.encryption_info,\n as_lines or rp.plaintext)\n\n def base64_segment(self, dec_start, dec_end, skip, line_len, line_end = 2):\n \"\"\"\n Given the start and end index of a desired segment of decoded data,\n this function finds smallest segment of an encoded base64 array that\n when decoded will include the desired decoded segment.\n It's assumed that the base64 data has a uniform line structure of\n line_len encoded characters including line_end eol characters,\n and that there are skip header characters preceding the base64 data.\n \"\"\"\n enc_start = 4*(dec_start/3)\n dec_skip = dec_start - 3*enc_start/4\n enc_start += line_end*(enc_start/(line_len-line_end))\n enc_end = 4*(dec_end/3)\n enc_end += line_end*(enc_end/(line_len-line_end))\n\n return enc_start, enc_end, dec_skip\n\n def pgp_packet_hdr_parse(self, header, prev_partial = False):\n \"\"\"\n Parse the header of a PGP packet to get the packet type, header length,\n and data length. Extra trailing characters in header are ignored.\n prev_partial indicates that the previous packet was a partial packet.\n An illegal header returns type -1, lengths 0.\n Header format is defined in RFC4880 section 4.\n \"\"\"\n hdr = bytearray(header.ljust( 6, chr(0)))\n if not prev_partial:\n hdr_len = 1\n else:\n hdr[1:] = hdr # Partial block headers don't have a tag\n hdr[0] = 0 # Insert a dummy tag.\n hdr_len = 0\n is_partial = False\n\n if prev_partial or (hdr[0] & 0xC0) == 0xC0:\n # New format packet\n ptag = hdr[0] & 0x3F\n body_len = hdr[1]\n lengthtype = 0\n hdr_len += 1\n if body_len < 192:\n pass\n elif body_len <= 223:\n hdr_len += 1\n body_len = ((body_len - 192) << 8) + hdr[2] + 192\n elif body_len == 255:\n hdr_len += 4\n body_len = ( (hdr[2] << 24) + (hdr[3] << 16) +\n (hdr[4] << 8) + hdr[5] )\n else:\n # Partial packet headers are only legal for data packets.\n if not prev_partial and not ptag in {8,9,11,18}:\n return (-1, 0, 0, False)\n # Could do extra testing here.\n is_partial = True\n body_len = 1 << (hdr[1] & 0x1F)\n\n elif (hdr[0] & 0xC0) == 0x80:\n # Old format packet\n ptag = (hdr[0] & 0x3C) >> 2\n lengthtype = hdr[0] & 0x03\n if lengthtype < 3:\n hdr_len = 2\n body_len = hdr[1]\n if lengthtype > 0:\n hdr_len = 3\n body_len = (body_len << 8) + hdr[2]\n if lengthtype > 1:\n hdr_len = 5\n body_len = (\n (body_len << 16) + (hdr[3] << 8) + hdr[4] )\n else:\n # Kludgy extra test for compressed packets w/ \"unknown\" length\n # gpg generates these in signed-only files. Check for valid\n # compression algorithm id to minimize false positives.\n if ptag != 8 or (hdr[1] < 1 or hdr[1] > 3):\n return (-1, 0, 0, False)\n hdr_len = 1\n body_len = -1\n else:\n return (-1, 0, 0, False)\n\n if hdr_len > len(header):\n return (-1, 0, 0, False)\n\n return ptag, hdr_len, body_len, is_partial\n\n\n def sniff(self, data, encoding = None):\n \"\"\"\n Checks arbitrary data to see if it is a PGP object and returns a set\n that indicates the kind(s) of object found. The names of the set\n elements are based on RFC3156 content types with 'pgp-' stripped so\n they can be used in sniffers for other protocols, e.g. S/MIME.\n There are additional set elements 'armored' and 'unencrypted'.\n\n This code should give no false negatives, but may give false positives.\n For efficient handling of encoded data, only small segments are decoded.\n Armored files are detected by their armor header alone.\n Non-armored data is detected by looking for a sequence of valid PGP\n packet headers.\n \"\"\"\n\n found = set()\n is_base64 = False\n is_quopri = False\n line_len = 0\n line_end = 1\n enc_start = 0\n enc_end = 0\n dec_start = 0\n skip = 0\n ptag = 0\n hdr_len = 0\n body_len = 0\n partial = False\n offset_enc = 0\n offset_dec = 0\n offset_packet = 0\n\n # Identify encoding and base64 line length.\n if encoding and encoding.lower() == 'base64':\n line_len = data.find('\\n') + 1 # Assume uniform length\n if line_len < 0:\n line_len = len(data)\n elif line_len > 1 and data[line_len-2] == '\\r':\n line_end = 2\n if line_len - line_end > 76: # Maximum per RFC2045 6.8\n return found\n enc_end = line_len\n try:\n segment = base64.b64decode(data[enc_start:enc_end])\n except TypeError:\n return found\n is_base64 = True\n\n elif encoding and encoding.lower() == 'quoted-printable':\n # Can't selectively decode quopri because encoded length is data\n # dependent due to escapes! Just decode one medium length segment.\n # This is enough to contain the first few packets of a long file.\n try:\n segment = quopri.decodestring(data[0:1500])\n except TypeError:\n return found # *** ? Docs don't list exceptions\n is_quopri = True\n else:\n line_len = len(data)\n segment = data # *** Shallow copy?\n\n if not segment:\n found = set()\n elif not (ord(segment[0]) & 0x80):\n # Not a PGP packet header if MSbit is 0. Check for armoured data.\n found.add('armored')\n if segment.startswith(self.ARMOR_BEGIN_SIGNED):\n # Clearsigned\n found.add('unencrypted')\n found.add('signature')\n elif segment.startswith(self.ARMOR_BEGIN_SIGNATURE):\n # Detached signature\n found.add('signature')\n elif segment.startswith(self.ARMOR_BEGIN_ENCRYPTED):\n # PGP uses the same armor header for encrypted and signed only\n # Fortunately gpg --decrypt handles both!\n found.add('encrypted')\n elif segment.startswith(self.ARMOR_BEGIN_PUB_KEY):\n found.add('key')\n else:\n found = set()\n else:\n # Could be PGP packet header. Check for sequence of legal headers.\n while skip < len(segment) and body_len != -1:\n # Check this packet header.\n prev_partial = partial\n ptag, hdr_len, body_len, partial = (\n self.pgp_packet_hdr_parse(segment[skip:], prev_partial))\n\n if prev_partial or partial:\n pass\n elif ptag == 11:\n found.add('unencrypted') # Literal Data\n elif ptag == 1:\n found.add('encrypted') # Encrypted Session Key\n elif ptag == 9:\n found.add('encrypted') # Symmetrically Encrypted Data\n elif ptag == 18:\n found.add('encrypted') # Symmetrically Encrypted & MDC\n elif ptag == 2:\n found.add('signature') # Signature\n elif ptag == 4:\n found.add('signature') # One-Pass Signature\n elif ptag == 6:\n found.add('key') # Public Key\n elif ptag == 14:\n found.add('key') # Public Subkey\n elif ptag == 8: # Compressed Data Packet\n # This is a kludge. Signed, non-encrypted files made by gpg\n # (but no other gpg files) consist of one compressed data\n # packet of unknown length which contains the signature\n # and data packets.\n # This appears to be an interpretation of RFC4880 2.3.\n # The compression prevents selective parsing of headers.\n # So such packets are assumed to be signed messages.\n if dec_start == 0 and body_len == -1:\n found.add('signature')\n found.add('unencrypted')\n elif ptag < 0 or ptag > 19:\n found = set()\n return found\n\n dec_start += hdr_len + body_len\n skip = dec_start\n if is_base64 and body_len != -1:\n enc_start, enc_end, skip = self.base64_segment(dec_start,\n dec_start + 6, 0, line_len, line_end )\n segment = base64.b64decode(data[enc_start:enc_end])\n\n if is_base64 and body_len != -1 and skip != len(segment):\n # End of last packet does not match end of data.\n found = set()\n return found\n\n def remove_armor(self, text):\n lines = text.strip().splitlines(True)\n if lines[0].startswith(self.ARMOR_BEGIN_SIGNED):\n for idx in reversed(range(0, len(lines))):\n if lines[idx].startswith(self.ARMOR_BEGIN_SIGNATURE):\n lines = lines[:idx]\n while lines and lines[0].strip():\n lines.pop(0)\n break\n return ''.join(lines).strip()\n\n def verify(self, data, signature=None):\n \"\"\"\n >>> g = GnuPG(None)\n >>> s = g.sign(\"Hello, World\", _from=\"smari@mailpile.is\",\n clearsign=True)[1]\n >>> g.verify(s)\n \"\"\"\n params = [\"--verify\"]\n if signature:\n sig = tempfile.NamedTemporaryFile()\n sig.write(signature)\n sig.flush()\n params.append(sig.name)\n params.append(\"-\")\n\n self.event.running_gpg(_('Checking signature in %d bytes of data'\n ) % len(data))\n ret, retvals = self.run(params, gpg_input=data, partial_read_ok=True)\n\n rp = GnuPGResultParser(debug=self.debug)\n return rp.parse([None, retvals]).signature_info\n\n def encrypt(self, data, tokeys=[], armor=True,\n sign=False, fromkey=None, throw_keyids=False):\n \"\"\"\n >>> g = GnuPG(None)\n >>> g.encrypt(\"Hello, World\", to=[\"smari@mailpile.is\"])[0]\n 0\n \"\"\"\n if tokeys:\n action = [\"--encrypt\", \"--yes\", \"--expert\",\n \"--trust-model\", \"always\"]\n for r in tokeys:\n action.append(\"--recipient\")\n action.append(r)\n action.extend([])\n self.event.running_gpg(_('Encrypting %d bytes of data to %s'\n ) % (len(data), ', '.join(tokeys)))\n else:\n action = [\"--symmetric\", \"--yes\", \"--expert\"]\n self.event.running_gpg(_('Encrypting %d bytes of data with password'\n ) % len(data))\n\n if armor:\n action.append(\"--armor\")\n if sign:\n action.append(\"--sign\")\n if sign and fromkey:\n action.append(\"--local-user\")\n action.append(fromkey)\n if throw_keyids:\n action.append(\"--throw-keyids\")\n if fromkey:\n self.prepare_passphrase(fromkey, signing=True)\n\n retvals = self.run(action, gpg_input=data,\n send_passphrase=(sign or not tokeys))\n\n return retvals[0], \"\".join(retvals[1][\"stdout\"])\n\n def sign(self, data,\n fromkey=None, armor=True, detach=True, clearsign=False,\n passphrase=None):\n \"\"\"\n >>> g = GnuPG(None)\n >>> g.sign(\"Hello, World\", fromkey=\"smari@mailpile.is\")[0]\n 0\n \"\"\"\n if passphrase is not None:\n self.passphrase = passphrase.get_reader()\n if fromkey and passphrase is None:\n self.prepare_passphrase(fromkey, signing=True)\n\n if detach and not clearsign:\n action = [\"--detach-sign\"]\n elif clearsign:\n action = [\"--clearsign\"]\n else:\n action = [\"--sign\"]\n if armor:\n action.append(\"--armor\")\n if fromkey:\n action.append(\"--local-user\")\n action.append(fromkey)\n\n self.event.running_gpg(_('Signing %d bytes of data with %s'\n ) % (len(data), fromkey or _('default')))\n retvals = self.run(action, gpg_input=data, send_passphrase=True)\n\n self.passphrase = None\n return retvals[0], \"\".join(retvals[1][\"stdout\"])\n\n def sign_key(self, keyid, signingkey=None):\n action = [\"--yes\", \"--sign-key\", keyid]\n if signingkey:\n action.insert(1, \"-u\")\n action.insert(2, signingkey)\n\n self.event.running_gpg(_('Signing key %s with %s'\n ) % (keyid, signingkey or _('default')))\n retvals = self.run(action, send_passphrase=True)\n\n return retvals\n\n def delete_key(self, key_fingerprint):\n cmd = ['--yes', '--delete-secret-and-public-key', key_fingerprint]\n return self.run(cmd)\n\n def recv_key(self, keyid,\n keyservers=DEFAULT_KEYSERVERS,\n keyserver_options=(DEFAULT_KEYSERVER_OPTIONS+DEFAULT_IMPORT_OPTIONS)):\n if not keyid[:2] == '0x':\n keyid = '0x%s' % keyid\n self.event.running_gpg(_('Downloading key %s from key servers'\n ) % (keyid))\n for keyserver in keyservers:\n cmd = ['--keyserver', keyserver,\n '--recv-key', self._escape_hex_keyid_term(keyid)]\n for opt in keyserver_options:\n cmd[2:2] = ['--keyserver-options', opt]\n retvals = self.run(cmd)\n if 'unsupported' not in ''.join(retvals[1][\"stdout\"]):\n break\n return self._parse_import(retvals[1][\"status\"])\n\n def parse_hpk_response(self, lines):\n results = {}\n lines = [x.strip().split(\":\") for x in lines]\n curpub = None\n for line in lines:\n if line[0] == \"info\":\n pass\n elif line[0] == \"pub\":\n curpub = line[1]\n validity = line[6]\n if line[5]:\n if int(line[5]) < time.time():\n validity += 'e'\n results[curpub] = {\n \"created\": datetime.fromtimestamp(int(line[4])),\n \"created_ts\": int(line[4]),\n \"keytype_name\": _(openpgp_algorithms.get(int(line[2]),\n 'Unknown')),\n \"keysize\": line[3],\n \"validity\": validity,\n \"uids\": [],\n \"fingerprint\": curpub\n }\n elif line[0] == \"uid\":\n email, name, comment = parse_uid(urllib.unquote(line[1]))\n results[curpub][\"uids\"].append({\"name\": name,\n \"email\": email,\n \"comment\": comment})\n return results\n\n def search_key(self, term,\n keyservers=DEFAULT_KEYSERVERS,\n keyserver_options=(DEFAULT_KEYSERVER_OPTIONS+DEFAULT_IMPORT_OPTIONS)):\n self.event.running_gpg(_('Searching for key for %s in key servers'\n ) % (term))\n for keyserver in keyservers:\n cmd = ['--keyserver', keyserver,\n '--fingerprint',\n '--search-key', self._escape_hex_keyid_term(term)]\n for opt in keyserver_options:\n cmd[2:2] = ['--keyserver-options', opt]\n retvals = self.run(cmd)\n if 'unsupported' not in ''.join(retvals[1][\"stdout\"]):\n break\n return self.parse_hpk_response(retvals[1][\"stdout\"])\n\n def get_pubkey(self, keyid):\n return self.export_pubkeys(selectors=[keyid])\n\n def get_minimal_key(self, key_id=None, user_id=None, armor=True):\n # Note: We are not stripping revoked subkeys, revocations are\n # rare but important. A more nuanced approach might only\n # include *recent* revocations, but we don't have the\n # tooling for that.\n args = [\n '--export-options', 'export-minimal',\n '--export-filter', 'drop-subkey=expired-t||disabled-t']\n selector = key_id or user_id\n if not selector:\n raise ValueError('Export what key?')\n if user_id:\n args.extend(['--export-filter', 'keep-uid=uid =~ %s' % user_id])\n return self.export_pubkeys(\n extra_args=args, armor=armor, selectors=[selector])\n\n def export_pubkeys(self, selectors=None, armor=True, extra_args=[]):\n self.event.running_gpg(_('Exporting keys %s from keychain'\n ) % (selectors,))\n retvals = self.run((extra_args or []) +\n (['--armor'] if armor else []) +\n (['--export']) +\n (selectors or []))[1][\"stdout\"]\n return \"\".join(retvals)\n\n def export_privkeys(self, selectors=None):\n retvals = self.run(['--armor',\n '--export-secret-keys'] + (selectors or [])\n )[1][\"stdout\"]\n return \"\".join(retvals)\n\n def address_to_keys(self, address):\n res = {}\n keys = self.list_keys(selectors=[address])\n for key, props in keys.iteritems():\n if any([x[\"email\"] == address for x in props[\"uids\"]]):\n res[key] = props\n\n return res\n\n def _escape_hex_keyid_term(self, term):\n \"\"\"Prepends a 0x to hexadecimal key ids.\n\n For example, D13C70DA is converted to 0xD13C70DA. This is required\n by version 2.x of GnuPG (and is accepted by 1.x).\n \"\"\"\n is_hex_keyid = False\n if len(term) == GPG_KEYID_LENGTH or len(term) == 2*GPG_KEYID_LENGTH:\n hex_digits = set(string.hexdigits)\n is_hex_keyid = all(c in hex_digits for c in term)\n\n if is_hex_keyid:\n return '0x%s' % term\n else:\n return term\n\n def chat(self, gpg_args, callback, *args, **kwargs):\n \"\"\"This lets a callback have a chat with the GPG process...\"\"\"\n gpg_args = (\n self.common_args(interactive=True, will_send_passphrase=True) + [\n # We may be interactive, but we're not a human!\n \"--no-tty\",\n \"--command-fd=0\",\n \"--status-fd=1\"\n ] + (gpg_args or []))\n\n proc = None\n try:\n # Here we go!\n self.debug('Running %s' % ' '.join(gpg_args))\n self.event.update_args(gpg_args)\n proc = Popen(gpg_args, stdin=PIPE, stdout=PIPE, stderr=PIPE,\n bufsize=0, long_running=True)\n\n return callback(proc, *args, **kwargs)\n finally:\n # Close this so GPG will terminate. This should already have\n # been done, but we're handling errors here...\n if proc and proc.stdin:\n proc.stdin.close()\n if proc:\n self.event.update_return_code(proc.wait())\n else:\n self.event.update_return_code(-1)\n\n\ndef GetKeys(gnupg, config, people):\n keys = []\n missing = []\n ambig = []\n\n # First, we go to the contact database and get a list of keys.\n for person in set(people):\n if '#' in person:\n keys.append(person.rsplit('#', 1)[1])\n else:\n vcard = config.vcards.get_vcard(person)\n if vcard:\n # It is the VCard's job to give us the best key first.\n lines = [vcl for vcl in vcard.get_all('KEY')\n if vcl.value.startswith('data:application'\n '/x-pgp-fingerprint,')]\n if len(lines) > 0:\n keys.append(lines[0].value.split(',', 1)[1])\n else:\n missing.append(person)\n else:\n missing.append(person)\n\n # Load key data from gnupg for use below\n if keys:\n all_keys = gnupg.list_keys(selectors=keys)\n else:\n all_keys = {}\n\n if missing:\n # Keys are missing, so we try to just search the keychain\n all_keys.update(gnupg.list_keys(selectors=missing))\n found = []\n for key_id, key in all_keys.iteritems():\n for uid in key.get(\"uids\", []):\n if uid.get(\"email\", None) in missing:\n missing.remove(uid[\"email\"])\n found.append(uid[\"email\"])\n keys.append(key_id)\n elif uid.get(\"email\", None) in found:\n ambig.append(uid[\"email\"])\n\n # Next, we go make sure all those keys are really in our keychain.\n fprints = all_keys.keys()\n for key in keys:\n key = key.upper()\n if key.startswith('0x'):\n key = key[2:]\n if key not in fprints:\n match = [k for k in fprints if k.endswith(key)]\n if len(match) == 0:\n missing.append(key)\n elif len(match) > 1:\n ambig.append(key)\n\n if missing:\n raise KeyLookupError(_('Keys missing for %s'\n ) % ', '.join(missing), missing)\n elif ambig:\n ambig = list(set(ambig))\n raise KeyLookupError(_('Keys ambiguous for %s'\n ) % ', '.join(ambig), ambig)\n return keys\n\n\nclass OpenPGPMimeSigningWrapper(MimeSigningWrapper):\n CONTAINER_PARAMS = (('micalg', 'pgp-sha512'),\n ('protocol', 'application/pgp-signature'))\n SIGNATURE_TYPE = 'application/pgp-signature'\n SIGNATURE_DESC = 'OpenPGP Digital Signature'\n\n def crypto(self):\n return GnuPG(self.config, event=self.event)\n\n def get_keys(self, who):\n return GetKeys(self.crypto(), self.config, who)\n\n\nclass OpenPGPMimeEncryptingWrapper(MimeEncryptingWrapper):\n CONTAINER_PARAMS = (('protocol', 'application/pgp-encrypted'), )\n ENCRYPTION_TYPE = 'application/pgp-encrypted'\n ENCRYPTION_VERSION = 1\n\n # FIXME: Define _encrypt, allow throw_keyids\n\n def crypto(self):\n return GnuPG(self.config, event=self.event)\n\n def get_keys(self, who):\n return GetKeys(self.crypto(), self.config, who)\n\n\nclass OpenPGPMimeSignEncryptWrapper(OpenPGPMimeEncryptingWrapper):\n CONTAINER_PARAMS = (('protocol', 'application/pgp-encrypted'), )\n ENCRYPTION_TYPE = 'application/pgp-encrypted'\n ENCRYPTION_VERSION = 1\n\n def crypto(self):\n return GnuPG(self.config)\n\n def _encrypt(self, message_text, tokeys=None, armor=False):\n from_key = self.get_keys([self.sender])[0]\n # FIXME: Allow throw_keyids here.\n return self.crypto().encrypt(message_text,\n tokeys=tokeys, armor=True,\n sign=True, fromkey=from_key)\n\n def _update_crypto_status(self, part):\n part.signature_info.part_status = 'verified'\n part.encryption_info.part_status = 'decrypted'\n\n\nclass GnuPGExpectScript(threading.Thread):\n STARTUP = 'Startup'\n START_GPG = 'Start GPG'\n FINISHED = 'Finished'\n SEND_PASSPHRASE = '!!>STDIN>> [Passphrase from secure passphrase store]')\n reader = self.sps.get_reader()\n while True:\n c = reader.read()\n if c != '':\n proc.stdin.write(c)\n else:\n proc.stdin.write('\\n')\n break\n elif line == self.SEND_EOF:\n self.gnupg.debug('>>STDIN>> [EOF]')\n proc.stdin.close()\n elif line is not None:\n self.gnupg.debug('>>STDIN>> \"%s\"' % line)\n proc.stdin.write(line.encode('utf-8'))\n proc.stdin.write('\\n')\n\n def _expecter(self, proc, exp, timebox):\n while timebox[0] > 0:\n read_char = proc.stdout.read(1)\n if read_char:\n self.before += read_char\n if exp in self.before:\n if exp:\n self.gnupg.debug('==Found: %s' % exp)\n self.before = self.before.split(exp)[0]\n return True\n elif read_char == '\\n':\n self.gnupg.debug('< 0) else self.DEFAULT_TIMEOUT\n timebox = [timeout]\n self.before = ''\n try:\n if not exp:\n return True\n self.gnupg.debug('==Expect(%ss): %s' % (timeout, exp))\n if RunTimed(timeout, self._expecter, proc, exp, timebox):\n return True\n else:\n raise TimedOut()\n except TimedOut:\n timebox[0] = 0\n self.gnupg.debug('Timed out')\n print('Boo! %s not found in %s' % (exp, self.before))\n raise\n\n def run_script(self, proc, script):\n for exp, rpl, tmo, state in script:\n self.expect_exact(proc, exp, timeout=tmo)\n if rpl:\n self.sendline(proc, (rpl % self.variables).strip())\n if state:\n self.set_state(state)\n stderr = proc.stderr.read()\n if stderr:\n self.gnupg.debug('<= (2, 1, 12):\n # Version 2.1.12 was the first version with the key generation\n # and --pinentry=loopback semantics we require. We just don't\n # even try with older versions.\n if variables.get('keytype') == consts.KEYTYPE_CURVE25519:\n return GnuPG21Curve25519KeyGenerator(gnupg, **kwargs)\n else:\n return GnuPG21RSAKeyGenerator(gnupg, **kwargs)\n\n else:\n return GnuPGDummyKeyGenerator(gnupg, **kwargs)\n\n\n# Reset our translation variable\n_ = gettext\n\n## Include the SKS keyserver certificates here ##\nKEYSERVER_CERTIFICATE=\"\"\"\n-----BEGIN CERTIFICATE-----\nMIIFizCCA3OgAwIBAgIJAK9zyLTPn4CPMA0GCSqGSIb3DQEBBQUAMFwxCzAJBgNV\nBAYTAk5PMQ0wCwYDVQQIDARPc2xvMR4wHAYDVQQKDBVza3Mta2V5c2VydmVycy5u\nZXQgQ0ExHjAcBgNVBAMMFXNrcy1rZXlzZXJ2ZXJzLm5ldCBDQTAeFw0xMjEwMDkw\nMDMzMzdaFw0yMjEwMDcwMDMzMzdaMFwxCzAJBgNVBAYTAk5PMQ0wCwYDVQQIDARP\nc2xvMR4wHAYDVQQKDBVza3Mta2V5c2VydmVycy5uZXQgQ0ExHjAcBgNVBAMMFXNr\ncy1rZXlzZXJ2ZXJzLm5ldCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC\nggIBANdsWy4PXWNUCkS3L//nrd0GqN3dVwoBGZ6w94Tw2jPDPifegwxQozFXkG6I\n6A4TK1CJLXPvfz0UP0aBYyPmTNadDinaB9T4jIwd4rnxl+59GiEmqkN3IfPsv5Jj\nMkKUmJnvOT0DEVlEaO1UZIwx5WpfprB3mR81/qm4XkAgmYrmgnLXd/pJDAMk7y1F\n45b5zWofiD5l677lplcIPRbFhpJ6kDTODXh/XEdtF71EAeaOdEGOvyGDmCO0GWqS\nFDkMMPTlieLA/0rgFTcz4xwUYj/cD5e0ZBuSkYsYFAU3hd1cGfBue0cPZaQH2HYx\nQk4zXD8S3F4690fRhr+tki5gyG6JDR67aKp3BIGLqm7f45WkX1hYp+YXywmEziM4\naSbGYhx8hoFGfq9UcfPEvp2aoc8u5sdqjDslhyUzM1v3m3ZGbhwEOnVjljY6JJLx\nMxagxnZZSAY424ZZ3t71E/Mn27dm2w+xFRuoy8JEjv1d+BT3eChM5KaNwrj0IO/y\nu8kFIgWYA1vZ/15qMT+tyJTfyrNVV/7Df7TNeWyNqjJ5rBmt0M6NpHG7CrUSkBy9\np8JhimgjP5r0FlEkgg+lyD+V79H98gQfVgP3pbJICz0SpBQf2F/2tyS4rLm+49rP\nfcOajiXEuyhpcmzgusAj/1FjrtlynH1r9mnNaX4e+rLWzvU5AgMBAAGjUDBOMB0G\nA1UdDgQWBBTkwyoJFGfYTVISTpM8E+igjdq28zAfBgNVHSMEGDAWgBTkwyoJFGfY\nTVISTpM8E+igjdq28zAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4ICAQAR\nOXnYwu3g1ZjHyley3fZI5aLPsaE17cOImVTehC8DcIphm2HOMR/hYTTL+V0G4P+u\ngH+6xeRLKSHMHZTtSBIa6GDL03434y9CBuwGvAFCMU2GV8w92/Z7apkAhdLToZA/\nX/iWP2jeaVJhxgEcH8uPrnSlqoPBcKC9PrgUzQYfSZJkLmB+3jEa3HKruy1abJP5\ngAdQvwvcPpvYRnIzUc9fZODsVmlHVFBCl2dlu/iHh2h4GmL4Da2rRkUMlbVTdioB\nUYIvMycdOkpH5wJftzw7cpjsudGas0PARDXCFfGyKhwBRFY7Xp7lbjtU5Rz0Gc04\nlPrhDf0pFE98Aw4jJRpFeWMjpXUEaG1cq7D641RpgcMfPFvOHY47rvDTS7XJOaUT\nBwRjmDt896s6vMDcaG/uXJbQjuzmmx3W2Idyh3s5SI0GTHb0IwMKYb4eBUIpQOnB\ncE77VnCYqKvN1NVYAqhWjXbY7XasZvszCRcOG+W3FqNaHOK/n/0ueb0uijdLan+U\nf4p1bjbAox8eAOQS/8a3bzkJzdyBNUKGx1BIK2IBL9bn/HravSDOiNRSnZ/R3l9G\nZauX0tu7IIDlRCILXSyeazu0aj/vdT3YFQXPcvt5Fkf5wiNTo53f72/jYEJd6qph\nWrpoKqrwGwTpRUCMhYIUt65hsTxCiJJ5nKe39h46sg==\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB\nhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV\nBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5\nMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT\nEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR\nQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh\ndGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR\n6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X\npz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC\n9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV\n/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf\nZd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z\n+pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w\nqP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah\nSL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC\nu9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf\nFobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq\ncrxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E\nFgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB\n/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl\nwFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM\n4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV\n2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna\nFxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ\nCuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK\nboHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke\njkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL\nS0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb\nQOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl\n0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB\nNVOFBkpdn627G190\n-----END CERTIFICATE-----\n\"\"\"\n" }, { "path": "mailpile/crypto/keyinfo.py", "content": "from __future__ import print_function\nimport time\nimport traceback\n\nimport pgpdump\nimport pgpdump.packet\nfrom pgpdump.utils import PgpdumpException, get_int4\nfrom mailpile.util import dict_merge\n\n\n# Patch pgpdump so it stops crashing on weird public keys #####################\n\ndef monkey_patch_pgpdump():\n # Add Algorithm 22 to the lookup table\n pgpdump.packet.AlgoLookup.pub_algorithms[22] = 'EdDSA'\n\n # Patch the key parser to just silently ignore strange keys\n orig_pkm = pgpdump.packet.PublicKeyPacket.parse_key_material\n\n def _patched_pkm(self, offset):\n try:\n return orig_pkm(self, offset)\n except PgpdumpException:\n return offset\n pgpdump.packet.PublicKeyPacket.parse_key_material = _patched_pkm\n\n\n# FIXME: Perhaps we should be checking pgpdump versions? But most of\n# these are actually API changes, not just bugfixes. It's likely\n# that versions 1.6+ will continue to throw exceptsions on \"unknown\" key\n# types... if/when 1.6 or 2.x get released, we'll just have to revisit\n# this logic.\nmonkey_patch_pgpdump()\n\n\n# Classes for storing PGP key info ############################################\n\nclass ustr(str):\n def __new__(cls, content):\n return super(ustr, cls).__new__(cls, content.upper())\n\n\nclass RestrictedDict(dict):\n KEYS = {}\n\n @classmethod\n def prep_properties(cls):\n def mk_prop(k):\n return property(lambda s: s[k], lambda s, v: s.__setitem__(k, v))\n for k in cls.KEYS:\n setattr(cls, k, mk_prop(k))\n\n def __init__(self, *args, **kwargs):\n dict.__init__(self, *args, **kwargs)\n for k, (t, d) in self.KEYS.items():\n if k not in self:\n if t in (list, dict):\n self[k] = t()\n else:\n self[k] = d\n\n def keys(self):\n kl = list(dict.keys(self))\n for dk in (k for k in self.KEYS if k not in kl):\n kl.append(dk)\n return sorted(kl)\n\n def __setitem__(self, item, value):\n if item[:1] != '_':\n if item not in self.KEYS:\n raise KeyError('Invalid key: %s' % item)\n if not isinstance(value, self.KEYS[item][0]):\n try:\n if isinstance(value, unicode):\n # Value is unicode, we want other: encode, convert\n value = self.KEYS[item][0](value.encode('utf-8'))\n elif isinstance(value, str):\n # Value is not unicode, we want unicode: decode, convert\n value = self.KEYS[item][0](value.decode('utf-8'))\n else:\n # Neither unicode nor string, just try to convert\n value = self.KEYS[item][0](value)\n except (TypeError, ValueError):\n raise TypeError(\n 'Bad type for %s: %s (want %s)'\n % (item, value, self.KEYS[item][0].__name__))\n dict.__setitem__(self, item, value)\n\n def __getitem__(self, item):\n if item[:1] == '_':\n return dict.__getitem__(self, item)\n else:\n return dict.get(self, item, self.KEYS[item][1])\n\n\nclass KeyUID(RestrictedDict):\n KEYS = {\n 'name': (unicode, ''),\n 'email': (str, ''),\n 'comment': (unicode, '')}\n\n def __repr__(self):\n parts = []\n if self['name']:\n parts.append(self['name'])\n if self['email']:\n parts.append('<%s>' % self['email'])\n if self['comment']:\n parts.append('(%s)' % self['comment'])\n return ' '.join(parts)\n\n\nclass KeyInfo(RestrictedDict):\n KEY_TRUSTED_CODES = ('u', 'f') # Note: Ignoring marginal keys\n KEY_INVALID_CODES = ('i', 'd', 'e', 'r', 'n')\n KEYS = {\n 'fingerprint': (ustr, 'MISSING'),\n 'capabilities': (str, ''),\n 'keytype_name': (str, 'unknown'),\n 'keytype_code': (int, 0),\n 'keysize': (int, 0),\n 'created': (int, 0),\n 'expires': (int, 0),\n 'validity': (str, '?'),\n 'key_source': (str, None),\n 'uids': (list, None),\n 'subkeys': (list, None),\n 'is_subkey': (bool, False),\n 'have_secret': (bool, False),\n 'on_keychain': (bool, False),\n 'in_vcards': (bool, False)}\n\n expired = property(lambda k: time.time() > k.expires > 0)\n\n is_usable = property(lambda k: (k.validity not in k.KEY_INVALID_CODES\n and not k.expired))\n\n can_encrypt = property(lambda k: ('e' in k.capabilities.lower()\n and k.is_usable))\n\n can_sign = property(lambda k: ('s' in k.capabilities.lower()\n and k.is_usable))\n\n def summary(self, full_fingerprint=False):\n \"\"\"\n Generate a short string summarizing the key's main properties: key ID,\n UIDs, expiration date, algorithm, size, capabilities, and validity.\n\n Note: If summary ends with !, the key is invalid/unusable.\n \"\"\"\n now = time.time()\n emails = ','.join(sorted([u.email for u in self.uids if u.email]))\n return '%s%s%s/%s%s/%s%s' % (\n self.fingerprint[-(9999 if full_fingerprint else 16):],\n ('=%s' % emails) if emails else '',\n ('<%x' % self.expires) if self.expires else '',\n self.keytype_name[:3],\n self.keysize,\n self.capabilities,\n ('' if self.is_usable else '!'))\n\n def __repr__(self):\n if self.is_subkey:\n return self.summary()\n return '{ %s }' % '\\n '.join(\n '%-12s = %s' % (k, self[k])\n for k in self.keys() if self[k] is not None)\n\n def ensure_autocrypt_uid(keyinfo, ac_uid):\n \"\"\"Ensure we include the email from the Autocrypt header in a UID.\"\"\"\n if keyinfo.is_subkey:\n return\n found = 0\n for uid in keyinfo.uids:\n if uid.email == ac_uid.email:\n uid.comment = uid.comment + '(Autocrypt)'\n found += 1\n if not found:\n keyinfo.uids += [ac_uid]\n\n def add_subkey_capabilities(keyinfo, now=None):\n \"\"\"Make key \"inherit\" the capabilities of any un-expired subkeys.\"\"\"\n now = now or time.time()\n key_caps = set(c for c in keyinfo.capabilities\n if c in ('c', 'e', 's'))\n combined_caps = set(c.upper() for c in key_caps)\n for subkey in keyinfo.subkeys:\n if not (0 < subkey.expires < now):\n combined_caps |= set(c.upper() for c in subkey.capabilities)\n keyinfo.capabilities = '%s%s' % (\n ''.join(sorted(list(combined_caps))),\n ''.join(sorted(list(key_caps))))\n\n def synthesize_validity(keyinfo, now=None):\n \"\"\"Synthesize key validity property.\"\"\"\n # FIXME: Revocations?\n now = now or time.time()\n if (0 < keyinfo.expires < now\n and keyinfo.validity not in keyinfo.KEY_INVALID_CODES):\n keyinfo.validity = 'e'\n\n def recalculate_expiration(keyinfo, now=None):\n \"\"\"Adjust the main expiration date to take subkeys into account.\"\"\"\n now = now or time.time()\n\n # For each capability, figure out what is the latest expiration date\n # provided by a subkey for that capability.\n expirations = {}\n for cap in set(c for c in keyinfo.capabilities if c in ('C', 'E', 'S')):\n for subkey in keyinfo.subkeys:\n if subkey.expires and not (0 < subkey.expires < now):\n expirations[cap] = max(subkey.expires, expirations.get(cap, 0))\n\n for cap in expirations:\n # If the subkey is not expired, and provides a capability our\n # main key doesn't have, then its expiration date matters.\n if cap.lower() not in keyinfo.capabilities:\n keyinfo.expires = min(subkey.expires, keyinfo.expires)\n\n @classmethod\n def FromGPGI(cls, gpgi_keyinfo):\n mki = cls(\n created=int(gpgi_keyinfo.get(\"creation_date_ts\",\n gpgi_keyinfo.get('created_ts', 0))),\n expires=int(gpgi_keyinfo.get(\"expiration_date_ts\", 0)),\n capabilities=gpgi_keyinfo.get(\"capabilities\", \"\"),\n have_secret=gpgi_keyinfo.get(\"secret\", False))\n for k in ('fingerprint', 'validity', 'keytype_name'):\n mki[k] = str(gpgi_keyinfo[k])\n for k in ('keysize', ):\n mki[k] = int(gpgi_keyinfo[k])\n for uid in gpgi_keyinfo.get('uids', []):\n mki.uids.append(KeyUID(\n name=uid.get(\"name\", \"\"),\n email=uid.get(\"email\", \"\"),\n comment=uid.get(\"comment\", \"\")))\n mki.capabilities = ''.join(sorted([c for c in mki.capabilities]))\n return mki\n\n\nclass MailpileKeyInfo(KeyInfo):\n KEYS = dict_merge(KeyInfo.KEYS, {\n 'vcards': (dict, None),\n 'origins': (list, None),\n 'is_autocrypt': (bool, False),\n 'is_gossip': (bool, False),\n 'is_preferred': (bool, False),\n 'is_pinned': (bool, False),\n 'scores': (dict, None),\n 'score_stars': (int, 0),\n 'score_reason': (unicode, None),\n 'score': (int, 0)})\n\n\nKeyUID.prep_properties()\nKeyInfo.prep_properties()\nMailpileKeyInfo.prep_properties()\n\n\ndef get_keyinfo(data, autocrypt_header=None,\n key_info_class=KeyInfo, key_uid_class=KeyUID,\n key_source=None):\n \"\"\"\n This method will parse a stream of OpenPGP packets into a list of KeyInfo\n objects.\n\n Note: Signatures are not validated, this code only parses the data.\n \"\"\"\n try:\n if \"-----BEGIN\" in data:\n ak = pgpdump.AsciiData(data)\n else:\n ak = pgpdump.BinaryData(data)\n packets = list(ak.packets())\n except (TypeError, IndexError, PgpdumpException):\n traceback.print_exc()\n return []\n\n def _unixtime(packet, seconds=0, days=0):\n return (packet.raw_creation_time\n + (days or 0) * 24 * 3600\n + (seconds or 0))\n\n results = []\n last_uid = key_uid_class() # Dummy\n last_key = key_info_class() # Dummy\n last_pubkeypacket = None\n main_key_id = None\n for m in packets:\n try:\n if isinstance(m, pgpdump.packet.PublicKeyPacket):\n size = str(int(1.024 *\n round(len('%x' % (m.modulus or 0)) / 0.256)))\n last_pubkeypacket = m\n last_key = key_info_class(\n key_source=key_source,\n fingerprint=m.fingerprint,\n keytype_name=m.pub_algorithm or '',\n keytype_code=m.raw_pub_algorithm,\n keysize=size)\n if isinstance(m, pgpdump.packet.PublicSubkeyPacket):\n last_key.is_subkey = True\n results[-1].subkeys.append(last_key)\n else:\n main_key_id = m.key_id\n results.append(last_key)\n\n # Older pgpdumps may fail here and cause traceback noise, but\n # the loop will limp onwards.\n last_key.created = _unixtime(m)\n if m.raw_days_valid > 0:\n last_key.expires = _unixtime(m, days=m.raw_days_valid)\n if last_key.expires == last_key.created:\n last_key.expires = 0\n\n elif isinstance(m, pgpdump.packet.UserIDPacket) and results:\n last_uid = key_uid_class(name=m.user_name, email=m.user_email)\n last_key.uids.append(last_uid)\n\n elif isinstance(m, pgpdump.packet.SignaturePacket) and results:\n # Note: We don't actually check the signature; we trust\n # GnuPG will if we decide to use this key.\n if m.key_id == main_key_id:\n for s in m.subpackets:\n if s.subtype == 9:\n exp = _unixtime(last_pubkeypacket, seconds=get_int4(s.data, 0))\n last_key.expires = max(last_key.expires, exp)\n elif s.subtype == 27:\n caps = set(c for c in last_key.capabilities)\n for flag, c in ((0x01, 'c'), (0x02, 's'),\n (0x0C, 'e'), (0x20, 'a')):\n if s.data[0] & flag:\n caps.add(c)\n last_key.capabilities = ''.join(caps)\n\n except (TypeError, AttributeError, KeyError, IndexError, NameError):\n traceback.print_exc()\n\n autocrypt_uid = None\n if autocrypt_header:\n # The autocrypt spec tells us that the visible addr= attribute\n # overrides whatever is on the key itself, so we synthesize a\n # fake UID here so the info is correct in an Autocrypt context.\n autocrypt_uid = key_uid_class(\n email=autocrypt_header['addr'],\n comment='Autocrypt')\n\n now = time.time()\n for keyinfo in results:\n keyinfo.synthesize_validity(now=now)\n keyinfo.add_subkey_capabilities(now=now)\n keyinfo.recalculate_expiration(now=now)\n if autocrypt_uid is not None:\n keyinfo.ensure_autocrypt_uid(autocrypt_uid)\n\n return results\n\n\nif __name__ == \"__main__\":\n import sys\n\n for f in sys.argv[1:]:\n with open(f, 'r') as fd:\n keyinfo = get_keyinfo(fd.read())[0]\n print('%s' % keyinfo)\n print('%s' % keyinfo.summary(full_fingerprint=True))\n print('Is usable = %s, Can encrypt = %s, Can sign = %s' % (\n keyinfo.is_usable, keyinfo.can_encrypt, keyinfo.can_sign))\n print('')\n\n# EOF\n" }, { "path": "mailpile/crypto/mime.py", "content": "from __future__ import print_function\n# These are methods to do with MIME and crypto, implementing PGP/MIME.\n\nimport math\nimport random\nimport re\nimport StringIO\nimport email.parser\n\nfrom email import encoders\nfrom email.mime.multipart import MIMEMultipart\nfrom email.mime.base import MIMEBase\n\nfrom mailpile.crypto.state import EncryptionInfo, SignatureInfo\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailutils.generator import Generator\n\n\n##[ Common utilities ]#########################################################\n\ndef Normalize(payload):\n # http://tools.ietf.org/html/rfc3156 says we must:\n #\n # - use CRLF everywhere\n # - strip trailing whitespace\n # - end with a CRLF\n #\n # In particlar, the stripping of trailing whitespace seems (based on\n # experiments with mutt), to in practice mean \"strip trailing whitespace\n # off the last line\"...\n #\n text = re.sub(r'\\r?\\n', '\\r\\n', payload).rstrip(' \\t')\n if not text.endswith('\\r\\n'):\n text += '\\r\\n'\n return text\n\n\ndef MessageAsString(part, unixfrom=False):\n buf = StringIO.StringIO()\n Generator(buf).flatten(part, unixfrom=unixfrom)\n return Normalize(buf.getvalue()).replace('--\\r\\n--', '--\\r\\n\\r\\n--')\n\n\nclass EncryptionFailureError(ValueError):\n def __init__(self, message, to_keys):\n ValueError.__init__(self, message)\n self.to_keys = to_keys\n\n\nclass SignatureFailureError(ValueError):\n def __init__(self, message, from_key):\n ValueError.__init__(self, message)\n self.from_key = from_key\n\n\nDEFAULT_CHARSETS = ['utf-8', 'iso-8859-1']\n\n\ndef _decode_text_part(part, payload, charsets=None):\n for cs in (c for c in ([part.get_content_charset() or None] +\n (charsets or DEFAULT_CHARSETS)) if c):\n try:\n return cs, payload.decode(cs)\n except (UnicodeDecodeError, TypeError, LookupError):\n pass\n return '8bit', payload\n\n\ndef _update_text_payload(part, payload, charsets=None):\n if 'content-transfer-encoding' in part:\n # We want this recalculated by the charset setting below\n del part['content-transfer-encoding']\n charset, payload = _decode_text_part(part, payload, charsets=charsets)\n part.set_payload(payload, charset)\n\n\n##[ Methods for unwrapping encrypted parts ]###################################\n\n\ndef MimeAttachmentDisposition(part, kind, newpart):\n \"\"\"\n Create a Content-Disposition header for a processed attachment using the\n original file name if available from the PGP packet, otherwise try\n stripping the extension from the unprocessed attachment file name.\n \"\"\"\n # Delete embedded \\n and \\r (shouldn't get_filename() do this itself??).\n filename = part.get_filename().replace('\\n','').replace('\\r','')\n if filename:\n filename = filename.decode('utf-8', 'replace')\n part.encryption_info[\"description\"] = _(\"Decrypted: %s\") % filename\n\n if part.encryption_info.filename:\n newfilename = part.encryption_info.filename\n else:\n # get_filename() can parse quoted, folded and RFC2231 names.\n # If there's no filename in Content-Disposition it tries Content-Type.\n newfilename = filename\n if 'armored' in kind and newfilename.endswith('.asc'):\n newfilename = newfilename[:len(newfilename)-len('.asc')]\n elif newfilename.endswith('.gpg'):\n newfilename = newfilename[:len(newfilename)-len('.gpg')]\n\n # add_header() does quoting, folding, maybe someday RFC2231?.\n newpart.add_header('Content-Disposition', 'attachment',\n filename=newfilename.encode('utf-8'))\n\n\ndef MimeReplacePart(part, newpart, keep_old_headers=False):\n \"\"\"\n Replace a MIME part with new version (decrypted, signature verified, ... ).\n retaining headers from the old part that are not in the new part. The\n headers that would be overwritten will be renamed and kept if the\n keep_old_headers variable is set to a prefix string.\n\n MIME headers (Content-*) get special treatment.\n\n Returns a set of the headers that got copied from the new part.\n \"\"\"\n part.set_payload(newpart.get_payload())\n\n # Original MIME headers must go, whether we're replacing them or not.\n for hdr in [k for k in part.keys() if k.lower().startswith('content-')]:\n while hdr in part:\n del part[hdr]\n\n # If we're keeping the non-MIME old headers, make copies now before\n # they get deleted below.\n if keep_old_headers:\n if not isinstance(keep_old_headers, str):\n keep_old_headers = \"Old\"\n for h in newpart.keys():\n headers = (part.get_all(h) or [])\n if (len(headers) == 1) and (part[h] == newpart[h]):\n continue\n for v in headers:\n part.add_header('X-%s-%s' % (keep_old_headers, h), v)\n\n for h in newpart.keys():\n while h in part:\n del part[h]\n\n copied = set([])\n for h, v in newpart.items():\n part.add_header(h, v)\n if not h.lower().startswith('content-'):\n copied.add(h)\n\n return copied\n\n\ndef UnwrapMimeCrypto(part, protocols=None, psi=None, pei=None, charsets=None,\n unwrap_attachments=True, require_MDC=True,\n depth=0, sibling=0, efail_unsafe=False, allow_decrypt=True):\n \"\"\"\n This method will replace encrypted and signed parts with their\n contents and set part attributes describing the security properties\n instead.\n \"\"\"\n\n # Guard against maliciously constructed emails\n if depth > 6:\n return\n\n part.signature_info = SignatureInfo(parent=psi)\n part.encryption_info = EncryptionInfo(parent=pei)\n\n part.signed_headers = set([])\n part.encrypted_headers = set([])\n\n mimetype = part.get_content_type() or 'text/plain'\n disposition = part['content-disposition'] or \"\"\n encoding = part['content-transfer-encoding'] or \"\"\n\n # FIXME: Check the protocol. PGP? Something else?\n # FIXME: This is where we add hooks for other MIME encryption\n # schemes, so route to callbacks by protocol.\n crypto_cls = protocols['openpgp']\n\n if part.is_multipart():\n # Containers are by default not bubbly\n part.signature_info.bubbly = False\n part.encryption_info.bubbly = False\n\n if part.is_multipart() and mimetype == 'multipart/signed':\n try:\n boundary = part.get_boundary()\n payload, signature = part.get_payload()\n\n # The Python get_payload() method likes to rewrite headers,\n # which breaks signature verification. So we manually parse\n # out the raw payload here.\n head, raw_payload, junk = part.as_string(\n ).replace('\\r\\n', '\\n').split('\\n--%s\\n' % boundary, 2)\n\n part.signature_info = crypto_cls().verify(\n Normalize(raw_payload), signature.get_payload())\n part.signature_info.bubble_up(psi)\n\n # Reparent the contents up, removing the signature wrapper\n hdrs = MimeReplacePart(part, payload,\n keep_old_headers='PH-Renamed')\n part.signed_headers = hdrs\n\n # Try again, in case we just unwrapped another layer\n # of multipart/something.\n UnwrapMimeCrypto(part,\n protocols=protocols,\n psi=part.signature_info,\n pei=part.encryption_info,\n charsets=charsets,\n unwrap_attachments=unwrap_attachments,\n require_MDC=require_MDC,\n depth=depth+1, sibling=sibling,\n efail_unsafe=efail_unsafe,\n allow_decrypt=allow_decrypt)\n\n except (IOError, OSError, ValueError, IndexError, KeyError):\n part.signature_info = SignatureInfo()\n part.signature_info[\"status\"] = \"error\"\n part.signature_info.bubble_up(psi)\n\n elif part.is_multipart() and mimetype == 'multipart/encrypted':\n try:\n if not allow_decrypt:\n raise ValueError('Decryption forbidden, MIME structure is weird')\n preamble, payload = part.get_payload()\n (part.signature_info, part.encryption_info, decrypted) = (\n crypto_cls().decrypt(\n payload.as_string(), require_MDC=require_MDC))\n except (IOError, OSError, ValueError, IndexError, KeyError):\n part.encryption_info = EncryptionInfo()\n part.encryption_info[\"status\"] = \"error\"\n\n part.signature_info.bubble_up(psi)\n part.encryption_info.bubble_up(pei)\n\n if part.encryption_info['status'] == 'decrypted':\n newpart = email.parser.Parser().parsestr(decrypted)\n\n # Reparent the contents up, removing the encryption wrapper\n hdrs = MimeReplacePart(part, newpart,\n keep_old_headers='MH-Renamed')\n\n # Is there a Memory-Hole/Protected-Headers force-display part?\n pl = part.get_payload()\n if hdrs and isinstance(pl, list):\n if (pl[0]['content-type'].startswith('text/rfc822-headers;')\n and 'protected-headers' in pl[0]['content-type']):\n # Parse these headers as well and override the top level,\n # again. This is to be sure we see the same thing as\n # everyone else (same algo as enigmail).\n data = email.parser.Parser().parsestr(\n pl[0].get_payload(), headersonly=True)\n for h in data.keys():\n while h in part:\n del part[h]\n part[h] = data[h]\n hdrs.add(h)\n\n # Finally just delete the part, we're done with it!\n del pl[0]\n\n part.encrypted_headers = hdrs\n if part.signature_info[\"status\"] != 'none':\n part.signed_headers = hdrs\n\n # Try again, in case we just unwrapped another layer\n # of multipart/something.\n UnwrapMimeCrypto(part,\n protocols=protocols,\n psi=part.signature_info,\n pei=part.encryption_info,\n charsets=charsets,\n unwrap_attachments=unwrap_attachments,\n require_MDC=require_MDC,\n depth=depth+1, sibling=sibling,\n efail_unsafe=efail_unsafe,\n allow_decrypt=allow_decrypt)\n\n # If we are still multipart after the above shenanigans (perhaps due\n # to an error state), recurse into our subparts and unwrap them too.\n elif part.is_multipart():\n for count, sp in enumerate(part.get_payload()):\n # EFail mitigation: We decrypt attachments and the first part\n # of a nested multipart structure, but not any subsequent parts.\n # This allows rewriting of messages to *append* cleartext, but\n # disallows rewriting that pushes \"inline\" encrypted content\n # further down to where the recipient might not notice it.\n sp_disp = (unwrap_attachments and sp['content-disposition']) or \"\"\n allow_decrypt = (efail_unsafe\n or (count == sibling == 0)\n or sp_disp.startswith('attachment')) and allow_decrypt\n UnwrapMimeCrypto(sp,\n protocols=protocols,\n psi=part.signature_info,\n pei=part.encryption_info,\n charsets=charsets,\n unwrap_attachments=unwrap_attachments,\n require_MDC=require_MDC,\n depth=depth+1, sibling=count,\n efail_unsafe=efail_unsafe,\n allow_decrypt=allow_decrypt)\n\n elif disposition.startswith('attachment'):\n # The sender can attach signed/encrypted/key files without following\n # rules for naming or mime type.\n # So - sniff to detect parts that need processing and identify protocol.\n kind = ''\n for protocol in protocols:\n crypto_cls = protocols[protocol]\n kind = crypto_cls().sniff(part.get_payload(), encoding)\n if kind:\n break\n\n if unwrap_attachments and ('encrypted' in kind or 'signature' in kind):\n # Messy! The PGP decrypt operation is also needed for files which\n # are encrypted and signed, and files that are signed only.\n payload = part.get_payload( None, True )\n try:\n if not allow_decrypt:\n raise ValueError('Decryption forbidden, MIME structure is weird')\n (part.signature_info, part.encryption_info, decrypted) = (\n crypto_cls().decrypt(\n payload, require_MDC=require_MDC))\n except (IOError, OSError, ValueError, IndexError, KeyError):\n part.encryption_info = EncryptionInfo()\n part.encryption_info[\"status\"] = \"error\"\n\n part.signature_info.bubble_up(psi)\n part.encryption_info.bubble_up(pei)\n\n if (part.encryption_info['status'] == 'decrypted' or\n part.signature_info['status'] == 'verified'):\n\n # Force base64 encoding and application/octet-stream type\n newpart = MIMEBase('application', 'octet-stream')\n newpart.set_payload(decrypted)\n encoders.encode_base64(newpart)\n\n # Add Content-Disposition with appropriate filename.\n MimeAttachmentDisposition(part, kind, newpart)\n\n MimeReplacePart(part, newpart)\n\n # Is there another layer to unwrap?\n UnwrapMimeCrypto(part,\n protocols=protocols,\n psi=part.signature_info,\n pei=part.encryption_info,\n charsets=charsets,\n unwrap_attachments=unwrap_attachments,\n require_MDC=require_MDC,\n depth=depth+1, sibling=sibling,\n efail_unsafe=efail_unsafe,\n allow_decrypt=allow_decrypt)\n else:\n # FIXME: Best action for unsuccessful attachment processing?\n pass\n\n elif mimetype == 'text/plain':\n return UnwrapPlainTextCrypto(part,\n protocols=protocols,\n psi=psi,\n pei=pei,\n charsets=charsets,\n require_MDC=require_MDC,\n depth=depth+1, sibling=sibling,\n efail_unsafe=efail_unsafe,\n allow_decrypt=allow_decrypt)\n\n else:\n # FIXME: This is where we would handle cryptoschemes that don't\n # appear as multipart/...\n pass\n\n\n # Mix in our bubbles\n part.signature_info.mix_bubbles()\n part.encryption_info.mix_bubbles()\n\n # Bubble up!\n part.signature_info.bubble_up(psi)\n part.encryption_info.bubble_up(pei)\n\n\ndef UnwrapPlainTextCrypto(part, protocols=None, psi=None, pei=None,\n charsets=None, require_MDC=True,\n depth=0, sibling=0,\n efail_unsafe=False, allow_decrypt=True):\n \"\"\"\n This method will replace encrypted and signed parts with their\n contents and set part attributes describing the security properties\n instead.\n \"\"\"\n payload = part.get_payload(None, True).strip()\n si = SignatureInfo(parent=psi)\n ei = EncryptionInfo(parent=pei)\n for crypto_cls in protocols.values():\n crypto = crypto_cls()\n\n if (payload.startswith(crypto.ARMOR_BEGIN_ENCRYPTED) and\n payload.endswith(crypto.ARMOR_END_ENCRYPTED)):\n try:\n if not allow_decrypt:\n raise ValueError('Decryption forbidden, MIME structure is weird')\n si, ei, text = crypto.decrypt(payload, require_MDC=require_MDC)\n _update_text_payload(part, text, charsets=charsets)\n except (IOError, OSError, ValueError, IndexError, KeyError):\n ei = EncryptionInfo()\n ei[\"status\"] = \"error\"\n break\n\n elif (payload.startswith(crypto.ARMOR_BEGIN_SIGNED) and\n payload.endswith(crypto.ARMOR_END_SIGNED)):\n try:\n si = crypto.verify(payload)\n except (IOError, OSError, ValueError, IndexError, KeyError):\n si = SignatureInfo()\n si[\"status\"] = \"error\"\n _update_text_payload(part, crypto.remove_armor(payload),\n charsets=charsets)\n break\n\n part.signature_info = si\n part.signature_info.bubble_up(psi)\n part.encryption_info = ei\n part.encryption_info.bubble_up(pei)\n\n\n##[ Methods for stripping down message headers ]###############################\n\n\ndef ObscureSubject(subject):\n \"\"\"\n Replace the Subject line with something nondescript.\n \"\"\"\n return '...'\n\n\ndef ObscureNames(hdr):\n \"\"\"\n Remove names (leaving e-mail addresses) from the To: and Cc: headers.\n\n >>> ObscureNames(\"Bjarni R. E. , e@b.c (Elmer Boop)\")\n u', '\n\n \"\"\"\n from mailpile.mailutils.addresses import AddressHeaderParser\n return ', '.join('<%s>' % ai.address for ai in AddressHeaderParser(hdr))\n\n\ndef ObscureSender(sender):\n \"\"\"\n Remove as much metadata from the From: line as possible.\n \"\"\"\n return ObscureNames(sender)\n\n\ndef ObscureAllRecipients(sender):\n \"\"\"\n Remove all content from the To: and Cc: lines entirely.\n \"\"\"\n return \"recipients-suppressed;\"\n\n\n# A dictionary for use with MimeWrapper's obscured_headers parameter,\n# that will obscure only what is required from the public header.\nOBSCURE_HEADERS_REQUIRED = {\n 'autocrypt-gossip': lambda t: None}\n\n\n# A dictionary for use with MimeWrapper's obscured_headers parameter,\n# that will obscure as much of the metadata from the public header as\n# possible without breaking compatibility.\nOBSCURE_HEADERS_MILD = {\n 'subject': ObscureSubject,\n 'from': ObscureSender,\n 'sender': ObscureSender,\n 'reply-to': ObscureSender,\n 'to': ObscureNames,\n 'cc': ObscureNames,\n 'user-agent': lambda t: None,\n 'autocrypt-gossip': lambda t: None}\n\n\n# A dictionary for use with MimeWrapper's obscured_headers parameter,\n# that will obscure as much of the metadata from the public header as\n# possible. This is only useful with encrypted messages and will badly\n# break things unless the recipient is running an MUA that fully implements\n# Memory Hole / Protected Headers.\nOBSCURE_HEADERS_EXTREME = {\n 'subject': ObscureSubject,\n 'from': ObscureSender,\n 'sender': ObscureSender,\n 'reply-to': ObscureSender,\n 'to': ObscureAllRecipients,\n 'cc': lambda t: None,\n 'date': lambda t: None,\n 'in-reply-to': lambda t: None,\n 'references': lambda t: None,\n 'openpgp': lambda t: None,\n 'user-agent': lambda t: None,\n 'autocrypt-gossip': lambda t: None}\n\n\n##[ Methods for encrypting and signing ]#######################################\n\nclass MimeWrapper:\n CONTAINER_TYPE = 'multipart/mixed'\n CONTAINER_PARAMS = ()\n\n # These are the default \"memory hole\" settings; wrap/protect the\n # important user-visible headers.\n WRAPPED_HEADERS = ('subject', 'from', 'to', 'cc', 'date', 'user-agent',\n 'sender', 'reply-to', 'in-reply-to', 'references',\n 'openpgp')\n\n # Force-displayed headers; if these headers get obscured, add a\n # visible part that shows them to the user in legacy clients.\n FORCE_DISPLAY_HEADERS = ('subject', 'from', 'to', 'cc')\n\n # By default, no headers are obscured. That's a user preference,\n # since there's a trade-off between privacy and compatibility.\n OBSCURED_HEADERS = OBSCURE_HEADERS_REQUIRED\n\n def __init__(self, config,\n event=None, cleaner=None,\n sender=None, recipients=None,\n use_html_wrapper=False,\n wrapped_headers=None,\n obscured_headers=None):\n from mailpile.mailutils.emails import MakeBoundary\n self.config = config\n self.event = event\n self.sender = sender\n self.cleaner = cleaner\n self.recipients = recipients or []\n self.use_html_wrapper = use_html_wrapper\n self.container = c = MIMEMultipart(boundary=MakeBoundary())\n\n self.wrapped_headers = self.WRAPPED_HEADERS\n if wrapped_headers is not None:\n self.wrapped_headers = wrapped_headers or ()\n\n self.obscured_headers = self.OBSCURED_HEADERS\n if obscured_headers is not None:\n self.obscured_headers = obscured_headers or {}\n\n c.set_type(self.CONTAINER_TYPE)\n c.signature_info = SignatureInfo(bubbly=False)\n c.encryption_info = EncryptionInfo(bubbly=False)\n if self.cleaner:\n self.cleaner(self.container)\n for pn, pv in self.CONTAINER_PARAMS:\n self.container.set_param(pn, pv)\n\n def crypto(self):\n return NotImplementedError(\"Please override me\")\n\n def attach(self, part):\n c = self.container\n c.attach(part)\n\n if not hasattr(part, 'signature_info'):\n part.signature_info = SignatureInfo(parent=c.signature_info)\n part.encryption_info = EncryptionInfo(parent=c.encryption_info)\n else:\n part.signature_info.parent = c.signature_info\n part.signature_info.bubbly = True\n part.encryption_info.parent = c.encryption_info\n part.encryption_info.bubbly = True\n\n if self.cleaner:\n self.cleaner(part)\n del part['MIME-Version']\n return self\n\n def get_keys(self, people):\n return people\n\n def flatten(self, msg, unixfrom=False):\n return MessageAsString(msg, unixfrom=unixfrom)\n\n def get_only_text_part(self, msg):\n count = 0\n only_text_part = None\n for part in msg.walk():\n if part.is_multipart():\n continue\n count += 1\n mimetype = part.get_content_type() or 'text/plain'\n if mimetype != 'text/plain' or count != 1:\n return False\n else:\n only_text_part = part\n return only_text_part\n\n def wrap(self, msg, **kwargs):\n # Subclasses override\n return msg\n\n def prepare_wrap(self, msg):\n obscured = self.obscured_headers\n wrapped = self.wrapped_headers\n\n obscured_set = set([])\n to_delete = {}\n for (h, header_value) in msg.items():\n if not header_value:\n continue\n\n hl = h.lower()\n if hl == 'mime-version':\n to_delete[h] = True\n elif not hl.startswith('content-'):\n if hl in obscured:\n obscured_set.add(h)\n oh = obscured[hl](header_value)\n if oh:\n self.container.add_header(h, oh)\n else:\n self.container.add_header(h, header_value)\n if hl not in wrapped and hl not in obscured:\n to_delete[h] = True\n\n for h in to_delete:\n while h in msg:\n del msg[h]\n\n if hasattr(msg, 'signature_info'):\n self.container.signature_info = msg.signature_info\n self.container.encryption_info = msg.encryption_info\n\n return self.force_display_headers(msg, obscured_set)\n\n def force_display_headers(self, msg, obscured_set):\n # If we aren't changing the structure of the message (adding a\n # force-display part), we can just wrap the original and be done.\n if not [k for k in obscured_set\n if k.lower() in self.FORCE_DISPLAY_HEADERS]:\n return msg\n\n header_display = MIMEBase('text', 'rfc822-headers',\n protected_headers=\"v1\")\n header_display['Content-Disposition'] = 'inline'\n\n container = MIMEBase('multipart', 'mixed')\n container.attach(header_display)\n container.attach(msg)\n\n # Cleanup...\n for p in (msg, header_display, container):\n if 'MIME-Version' in p:\n del p['MIME-Version']\n if self.cleaner:\n self.cleaner(header_display)\n self.cleaner(msg)\n\n # NOTE: The copying happens at the end here, because we need the\n # cleaner (on msg) to have run first.\n display_headers = []\n to_delete = {}\n for h, v in msg.items():\n hl = h.lower()\n if not hl.startswith('content-') and not hl.startswith('mime-'):\n container.add_header(h, v)\n if hl in self.FORCE_DISPLAY_HEADERS and h in obscured_set:\n display_headers.append('%s: %s' % (h, v))\n to_delete[h] = True\n for h in to_delete:\n while h in msg:\n del msg[h]\n\n header_display.set_payload('\\r\\n'.join(reversed(display_headers)))\n\n return container\n\n\nclass MimeSigningWrapper(MimeWrapper):\n CONTAINER_TYPE = 'multipart/signed'\n CONTAINER_PARAMS = ()\n SIGNATURE_TYPE = 'application/x-signature'\n SIGNATURE_DESC = 'Abstract Digital Signature'\n\n def __init__(self, *args, **kwargs):\n MimeWrapper.__init__(self, *args, **kwargs)\n\n name = ('OpenPGP-digital-signature.html'\n if self.use_html_wrapper else\n 'OpenPGP-digital-signature.asc')\n self.sigblock = MIMEBase(*self.SIGNATURE_TYPE.split('/'))\n self.sigblock.set_param(\"name\", name)\n for h, v in ((\"Content-Description\", self.SIGNATURE_DESC),\n (\"Content-Disposition\",\n \"attachment; filename=\\\"%s\\\"\" % name)):\n self.sigblock.add_header(h, v)\n\n def _wrap_sig_in_html(self, sig):\n return (\n \"

%(title)s

\\n\\n%(description)s\\n\\n

\"\n \"
\\n%(sig)s\\n
\"\n \"%(ad)s.\"\n ) % self._wrap_sig_in_html_vars(sig)\n\n def _wrap_sig_in_html_vars(self, sig):\n return {\n # FIXME: We deliberately do not flag these messages for i18n\n # translation, since we rely on 7-bit content here so as\n # not to complicate the MIME structure of the message.\n \"title\": \"Digital Signature\",\n \"description\": (\n \"This is a digital signature, which can be used to verify\\n\"\n \"the authenticity of this message. You can safely discard\\n\"\n \"or ignore this file if your e-mail software does not\\n\"\n \"support digital signatures.\"),\n \"ad\": \"Generated by Mailpile\",\n \"ad_url\": \"https://www.mailpile.is/\", # FIXME: Link to help?\n \"sig\": sig}\n\n def _update_crypto_status(self, part):\n part.signature_info.part_status = 'verified'\n\n def wrap(self, msg, prefer_inline=False):\n from_key = self.get_keys([self.sender])[0]\n\n if prefer_inline:\n prefer_inline = self.get_only_text_part(msg)\n else:\n prefer_inline = False\n\n if prefer_inline is not False:\n message_text = Normalize(prefer_inline.get_payload(None, True)\n .strip() + '\\r\\n\\r\\n')\n status, sig = self.crypto().sign(message_text,\n fromkey=from_key,\n clearsign=True,\n armor=True)\n if status == 0:\n _update_text_payload(prefer_inline, sig)\n self._update_crypto_status(prefer_inline)\n return msg\n\n else:\n msg = self.prepare_wrap(msg)\n self.attach(msg)\n self.attach(self.sigblock)\n message_text = self.flatten(msg)\n status, sig = self.crypto().sign(message_text,\n fromkey=from_key, armor=True)\n if status == 0:\n if self.use_html_wrapper:\n sig = self._wrap_sig_in_html(sig)\n self.sigblock.set_payload(sig)\n self._update_crypto_status(self.container)\n return self.container\n\n raise SignatureFailureError(_('Failed to sign message!'), from_key)\n\n\nclass MimeEncryptingWrapper(MimeWrapper):\n CONTAINER_TYPE = 'multipart/encrypted'\n CONTAINER_PARAMS = ()\n ENCRYPTION_TYPE = 'application/x-encrypted'\n ENCRYPTION_VERSION = 0\n\n def __init__(self, *args, **kwargs):\n MimeWrapper.__init__(self, *args, **kwargs)\n\n self.version = MIMEBase(*self.ENCRYPTION_TYPE.split('/'))\n self.version.set_payload('Version: %s\\n' % self.ENCRYPTION_VERSION)\n for h, v in ((\"Content-Disposition\", \"attachment\"), ):\n self.version.add_header(h, v)\n\n self.enc_data = MIMEBase('application', 'octet-stream')\n for h, v in ((\"Content-Disposition\",\n \"attachment; filename=\\\"OpenPGP-encrypted-message.asc\\\"\"), ):\n self.enc_data.add_header(h, v)\n\n self.attach(self.version)\n self.attach(self.enc_data)\n\n def _encrypt(self, message_text, tokeys=None, armor=False):\n return self.crypto().encrypt(message_text,\n tokeys=tokeys, armor=True)\n\n def _update_crypto_status(self, part):\n part.encryption_info.part_status = 'decrypted'\n\n def _add_padding(self, text, chunksize=None):\n \"\"\"Add up to 16kB of whitespace to the end of a message as padding.\"\"\"\n if chunksize is None:\n chunksize = min(max(160, 2 ** int(math.log(len(text), 2))), 8192)\n pad = ('\\r\\n' + (' ' * 78)) * int(chunksize / 80)\n return (text\n + (pad if random.randint(0, 1) else '')\n + (pad[:chunksize - (len(text) % chunksize)]))\n\n def wrap(self, msg, prefer_inline=False):\n to_keys = set(self.get_keys(self.recipients + [self.sender]))\n\n if prefer_inline:\n prefer_inline = self.get_only_text_part(msg)\n else:\n prefer_inline = False\n\n if prefer_inline is not False:\n message_text = self._add_padding(\n Normalize(prefer_inline.get_payload(None, True)),\n # This padding is user facing, so don't overdo it.\n chunksize=160)\n status, enc = self._encrypt(message_text,\n tokeys=to_keys,\n armor=True)\n if status == 0:\n _update_text_payload(prefer_inline, enc)\n self._update_crypto_status(prefer_inline)\n return msg\n\n else:\n msg = self.prepare_wrap(msg)\n if self.cleaner:\n self.cleaner(msg)\n\n message_text = self._add_padding(self.flatten(msg))\n status, enc = self._encrypt(message_text,\n tokeys=to_keys,\n armor=True)\n if status == 0:\n self.enc_data.set_payload(enc)\n self._update_crypto_status(self.enc_data)\n return self.container\n\n raise EncryptionFailureError(_('Failed to encrypt message!'), to_keys)\n\n\nif __name__ == \"__main__\":\n import sys\n import doctest\n\n # FIXME: Add tests for the wrapping/unwrapping code. It's crazy that\n # we don't have such tests. :-(\n\n results = doctest.testmod(optionflags=doctest.ELLIPSIS)\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/crypto/records.py", "content": "#\n# NOTE: THIS CODE IS NOT BEING USED - YET.\n# IT IS HERE TO FACILITATE REVIEW, COMMENTS AND EXPERIMENTS.\n#\n# FIXME: Respond to following comment from Kai Michaelis\n#\n# The dictionaries in records.py completely lack any message\n# authentication and thus are vulnerable. The cryptographic checksums\n# of the plaintext do not prevent this. The comments mention that this\n# is intentional to limit destruction in case of bit flips. This is\n# misguided in two ways. First, it's the responsibility of the file\n# system and transport protocol to prevent these and second,\n# cryptographic checksums are not designed for error detecting or\n# correction.\n# ...\n# To derive keys for multiple shards in a EncryptedDictionary use a\n# proper KDF like HKDF [2] or a CPRNG like those defined in NIST\n# SP800-90a rev1 [3] (except Dual_EC DRBG :wink:).\n#\n# Full discussion here: https://github.com/mailpile/Mailpile/pull/1684\n#\n# FIXME: We've switched from AES-CBC to AES-CTR. We should review the\n# IV logic and make sure it is still sane given the different\n# properties of the underlying algorithms.\n#\n\"\"\"\nRecord-based AES encrypted data storage\n\nThis is a collection of modules designed to allow for easy Pythonic use\nof encrypted data stores.\n\nThe basics are provided by EncryptedRecordStore, which defines the\non-disk storage format (largely compatible with RFC2822) and takes care\nof the encryption and decryption of records.\n\nThe EncryptedBlobStore provides a subset of Pythonic list semantics,\nextending EncryptedRecordStore to allow for arbitrarily sized elements\nand splitting the storage accross multiple files to play nice with\nbackups, network-based storage and other environments where huge files\nmight be a problem.\n\nThe EncryptedDict provides a subset of Pythonic dict semantics, using a\nmixture of the above two classes.\n\nNotes:\n\n1. A simple SHA256 digest is used to derive an AES key. This would not\n be considered sufficient for low entropy (human generated) keys.\n2. All data storage is record based, which implies nontrivial amounts\n of disk space may be wasted. On the other hand, this is good for\n security as it obscures the size of the data being stored.\n3. The EncryptedDict uses the encryption key as a salt to ensure the\n hashing is not predictable to an attacker. Again, low entropy keys\n should be avoided.\n4. No provisions are made to make it possible to change keys.\n5. No buffering or caching of any kind is done.\n6. Deleting entries is NOT supported anywhere. Overwriting works.\n\nPerformance thoughts:\n\nThe key to performance of these algorithms will ultimately depend on how\nwell the OS caches data. In general we can help with that by encouraging\nhot spots, trying to cluster frequently used values together. The other\nway we can help is to minimize wasted space within the records\nthemselves, so the OS doesn't waste RAM caching junk.\n\nFor the metadata index, we expect hot spot clustering to focus around\nrecently received mail. The default sorts are by date and most of the\ntime users are reading or organizing recently received mail. So the OS\nshould have a relatively easy time effectively caching records for\ncurrent metadata.\n\nThe case for posting lists, which naturally live in an EncryptedDict is\ndifferent but also promising. In general, the keyword index will grow\nlinearly with the number of index messages; in particular each message\nID is unique and will generate one or more new keywords in the index.\nThus the keyword index will have a very long tail of rarely used, small\nentries. The expected performance of such entries (reads and writes) is\ndominated by the disk seeks times. For these entries, we can save at\nleast one disk seek by storing the values along with the keys, but that\nis about all we can do.\n\nConversely, some keywords will have a very high frequency; for example\nvirtually all English language messages will contain the word \"the\".\nThese common keywords will become hot spots during the indexing process,\nso causing them to cluster together will again let us play nice with the\noperating system caches. A basic strategy for this is to allow larger\nentries to bump smaller ones from the front of each hash bucket to later\nstages, as entry size correlates with keyword popularity. Another\noptimization which is out of scope for this module, is they should\ncompress well as they will contain long sequential runs of message IDs.\n\nFrom a frequency point of view, the middle-of-the-road keywords barely\nmatter; 94% of all keywords match fewer than 5 messages, about 0.16%\nmatch over 1000 messages. On average about 17 keywords are generated\nper message.\n\nDetailed search keyword stats:\n\n emails keywords __________ratios__________\n 2 4551468 88.21% 100.00% 88.21%\n 4 290686 5.63% 11.79% 93.84%\n 8 150924 2.92% 6.16% 96.77%\n 16 74239 1.44% 3.23% 98.21%\n 32 38341 0.74% 1.79% 98.95%\n 64 20552 0.40% 1.05% 99.35%\n 128 13178 0.26% 0.65% 99.60%\n 256 7731 0.15% 0.40% 99.75%\n 512 4640 0.09% 0.25% 99.84%\n 1024 2793 0.05% 0.16% 99.90%\n 2048 2141 0.04% 0.10% 99.94%\n >2048 3084 0.06% 0.06% 100.00%\n (sample size: ~300k emails)\n\n\"\"\"\nfrom __future__ import print_function\nfrom __future__ import absolute_import\nimport binascii\nimport hashlib\nimport os\nimport struct\nimport time\nimport threading\n\nfrom .aes_utils import getrandbits, aes_ctr_encrypt, aes_ctr_decrypt\n\n\nclass _SimpleList(object):\n \"\"\"Some syntactic sugar for listalikes\"\"\"\n def append(self, value):\n with self._lock:\n pos = len(self)\n self[pos] = value\n return pos\n\n def extend(self, values):\n for v in values:\n self.append(v)\n\n def __getslice__(self, i, j):\n return [self[v] for v in range(i, j)]\n\n def __iadd__(self, y):\n self.extend(y)\n\n def __iter__(self):\n return (self[v] for v in range(0, len(self)))\n\n def __reversed__(self):\n return (self[v] for v in reversed(range(0, len(self))))\n\n\nclass EncryptedRecordStore(_SimpleList):\n \"\"\"\n This is an on-disk AES encrypted record storage. Data is written\n out base64 encoded, with 64 characters per line + CRLF, as that is\n likely to make it in and out of IMAP servers or other mail stores\n unchanged, while giving us a round multiple of what both AES and\n Base64 can handle without padding.\n \"\"\"\n\n _HEADER = ('X-Mailpile-Encrypted-Records: v1\\r\\n'\n 'From: Mailpile \\r\\n'\n 'Subject: %(filename)s\\r\\n'\n 'cipher: aes-128-ctr\\r\\n'\n 'record-size: %(record_size)d\\r\\n'\n 'iv-seed: %(ivs)s\\r\\n'\n '\\r\\n')\n\n def __init__(self, fn, key,\n max_bytes=400, overwrite=False, default_value=None):\n key_hash = bytes(hashlib.sha256(key.encode('utf-8')).digest())\n\n self._NEWLINE = '\\r\\n'\n self._max_bytes = max_bytes\n self._calculate_constants()\n self._default_value = default_value\n\n self._iv_seed = getrandbits(48)\n self._iv_seed_random = '%x' % getrandbits(64)\n self._aes_key = key_hash[:16]\n self._header_data = {\n 'record_size': self._RECORD_SIZE,\n 'filename': fn,\n 'ivs': '%12.12x' % self._iv_seed\n }\n self._lock = threading.RLock()\n\n try:\n self._fd = open(fn, 'wb+' if overwrite else 'rb+')\n except (OSError, IOError):\n self._fd = open(fn, 'wb+')\n\n if not self._parse_header():\n self._write_header()\n if max_bytes > self._max_bytes:\n raise AssertionError('max_bytes mismatch')\n\n def _calculate_constants(self):\n # Calculate our constants! Magic numbers:\n # - 48 is how much data fits in 64 byte of base64\n # - 16 is the size of the IV\n # - 7 is the minimum size of our checksum\n self._RECORD_LINES = max(1, int((self._max_bytes + 7 + 16) / 48) + 1)\n self._RECORD_SIZE = self._RECORD_LINES * 48\n self._MAX_DATA_SIZE = self._RECORD_SIZE - (7 + 16)\n self._RECORD_LINE_BYTES = 64\n self._ZERO_RECORD = '\\0' * self._RECORD_SIZE\n self._RECORD_BYTES = self._RECORD_LINES * (self._RECORD_LINE_BYTES +\n len(self._NEWLINE))\n\n def _parse_header(self):\n try:\n with self._lock:\n self._fd.seek(0)\n header = self._fd.read(len(self._HEADER % self._header_data)\n + 1024)\n if not header:\n return False\n\n if '\\n' in header and self._NEWLINE not in header:\n self._NEWLINE = '\\n'\n header = header.split(self._NEWLINE + self._NEWLINE)[0]\n headers = dict(hl.strip().split(': ', 1)\n for hl in header.splitlines())\n\n self._header_skip = len(header) + len(self._NEWLINE) * 2\n self._iv_seed = long(headers['iv-seed'], 16) + 10240020\n self._max_bytes = int(headers['record-size']) - (7 + 16) - 1\n self._header_data = {\n 'record_size': int(headers['record-size']),\n 'filename': headers['Subject'],\n 'ivs': headers['iv-seed']\n }\n self._calculate_constants()\n return True\n\n except IOError:\n return False\n except (KeyError, ValueError, AssertionError):\n import traceback\n traceback.print_exc()\n return False\n\n def _write_header(self):\n with self._lock:\n self._header_data['ivs'] = '%12.12x' % self._iv_seed\n header = self._HEADER % self._header_data\n self._fd.seek(0)\n self._fd.write(header)\n self._fd.flush()\n self._header_skip = len(header)\n\n def close(self):\n self._write_header()\n self._fd.close()\n\n def _iv(self, pos):\n # Here we generate an IV that should never repeat: the first 6 bytes\n # are derived from a random counter created at program startup, the\n # rest is pseudorandom crap.\n with self._lock:\n self._iv_seed += 1\n self._iv_seed %= 0x1000000000000\n if (self._iv_seed % 123456) == 0:\n self._write_header()\n\n self._iv_seed_random = hashlib.sha512(self._iv_seed_random\n ).digest()\n\n iv = bytes(struct.pack(' self._MAX_DATA_SIZE:\n raise ValueError('Data too big for record')\n\n pos = int(pos)\n if pos < 0:\n raise KeyError('Negative record position')\n\n iv = self._iv(pos)\n\n # We're using MD5 as a checksum here to detect corruption.\n cks = hashlib.md5(data).hexdigest()\n\n # We use the checksum as padding, where we are guaranteed by the\n # assertion above to have room for at least 6 nybbles of the MD5.\n record = (data + ':' + cks + self._ZERO_RECORD\n )[:self._RECORD_SIZE - len(iv)]\n encrypted = (iv + aes_ctr_encrypt(self._aes_key, iv, record)\n ).encode('base64').replace('\\n', '')\n\n if len(encrypted) != self._RECORD_LINES * self._RECORD_LINE_BYTES:\n raise AssertionError('%d: <%s> %s != %d*%d' %\n (pos, encrypted, len(encrypted),\n self._RECORD_LINES, self._RECORD_LINE_BYTES))\n\n with self._lock:\n self._fd.seek(self._header_skip + pos * self._RECORD_BYTES)\n for i in range(0, self._RECORD_LINES):\n self._fd.write(encrypted[i * self._RECORD_LINE_BYTES :\n (i+1) * self._RECORD_LINE_BYTES])\n self._fd.write(self._NEWLINE)\n\n def save_zeros(self, pos, count):\n zrecord = (('A' * self._RECORD_LINE_BYTES) + self._NEWLINE\n ) * self._RECORD_LINES\n with self._lock:\n self._fd.seek(self._header_skip + pos * self._RECORD_BYTES)\n for i in range(0, count):\n self._fd.write(zrecord)\n\n def load_record(self, pos):\n pos = int(pos)\n if pos < 0:\n raise KeyError('Negative record position')\n\n with self._lock:\n self._fd.seek(self._header_skip + pos * self._RECORD_BYTES)\n try:\n encrypted = self._fd.read(self._RECORD_BYTES).decode('base64')\n except (IOError, binascii.Error):\n encrypted = ''\n\n if encrypted == '':\n if self._default_value is not None:\n return self._default_value\n raise KeyError('Failed to read at %s' % pos)\n if (len(encrypted) != self._RECORD_SIZE):\n raise KeyError('Incorrect record size %s at %s'\n % (len(encrypted), pos))\n\n iv, encrypted = encrypted[:16], encrypted[16:]\n if iv == ('\\x00' * 16) and self._default_value is not None:\n return self._default_value\n\n plaintext, checksum = aes_ctr_decrypt(self._aes_key, iv, encrypted\n ).rsplit(':', 1)\n\n checksum = bytes(checksum.replace('\\0', ''))\n if (len(checksum) < 6):\n raise ValueError('Checksum too short')\n if not hashlib.md5(plaintext).hexdigest().startswith(checksum):\n raise ValueError('Checksum mismatch')\n\n return plaintext\n\n def __getitem__(self, pos):\n try:\n return self.load_record(pos)\n except ValueError:\n raise KeyError(pos)\n\n def __setitem__(self, pos, data):\n return self.save_record(pos, data)\n\n def get(self, pos, default=None):\n try:\n return self[pos]\n except KeyError:\n return default\n\n def __len__(self):\n with self._lock:\n self._fd.seek(0, 2)\n size = self._fd.tell()\n return (size - self._header_skip) // self._RECORD_BYTES\n\n\nclass EncryptedBlobStore(_SimpleList):\n \"\"\"\n This is an on-disk variable-sized, sharded AES encrypted blob\n storage. It augments EncryptedRecordStore by placing bounds on\n how large each encrypted file will become and allows data blobs\n of arbitrary size by bumping \"large\" records to a secondary (or\n tertiary, ...) store.\n \"\"\"\n\n _BIG_POINTER = '\\0B->'\n\n def __init__(self, base_fn, key,\n max_bytes=400, shard_size=50000, big_ratio=10,\n overwrite=False):\n self._base_fn = base_fn\n self._key = key\n self._max_bytes = max(max_bytes, 50)\n self._shard_size = max(int(shard_size), 1024)\n self._big_ratio = float(big_ratio)\n self._overwrite = overwrite\n\n self._lock = threading.RLock()\n\n self._shards = []\n self._load_next_shard()\n while os.path.exists(self._next_shardname()):\n self._load_next_shard()\n\n self._big_map = {}\n self._big_shard = None\n\n s0 = property(lambda self: self._shards[0])\n\n def _big(self):\n with self._lock:\n if self._big_shard is None:\n self._big_shard = EncryptedBlobStore(\n '%s-b' % self._base_fn,\n self._key,\n max_bytes=self._max_bytes * self._big_ratio,\n shard_size=self._shard_size / self._big_ratio,\n big_ratio=self._big_ratio,\n overwrite=self._overwrite)\n return self._big_shard\n\n def _next_shardname(self):\n return '%s-%s' % (self._base_fn, len(self._shards) + 1)\n\n def _load_next_shard(self):\n with self._lock:\n self._shards.append(EncryptedRecordStore(\n self._next_shardname(),\n self._key,\n max_bytes=self._max_bytes,\n overwrite=self._overwrite))\n\n def __getitem__(self, pos):\n shard, pos = pos // self._shard_size, pos % self._shard_size\n value = self._shards[shard][pos]\n if value.startswith(self._BIG_POINTER):\n self._big_map[pos] = int(value[len(self._BIG_POINTER):], 16)\n return self._big()[self._big_map[pos]]\n else:\n return value\n\n def __setitem__(self, pos, data):\n shard, pos = pos // self._shard_size, pos % self._shard_size\n with self._lock:\n while shard >= len(self._shards):\n self._load_next_shard()\n if len(data) > self.s0._MAX_DATA_SIZE:\n with self._lock:\n bpos = self._big_map.get(pos, len(self._big()))\n self._big()[bpos] = data\n self._big_map[pos] = bpos\n data = '%s%x' % (self._BIG_POINTER, bpos)\n self._shards[shard][pos] = data\n\n def __len__(self):\n with self._lock:\n return (self._shard_size * (len(self._shards) - 1) +\n len(self._shards[-1]))\n\n def close(self):\n with self._lock:\n for s in self._shards:\n s.close()\n if self._big_shard is not None:\n self._big_shard.close()\n self._shards = []\n self._big_shard = None\n\n\nclass EncryptedUnicodeStore(EncryptedBlobStore):\n \"\"\"\n An EncryptedBlobStore that only works with unicode data.\n \"\"\"\n def __setitem__(self, pos, data):\n return EncryptedBlobStore.__setitem__(self, pos, data.encode('utf-8'))\n\n def __getitem__(self, pos):\n return EncryptedBlobStore.__getitem__(self, pos).decode('utf-8')\n\n\nclass EncryptedDict(object):\n \"\"\"\n This is a variable-sized, sharded AES encrypted dict. It uses\n EncryptedRecordStore for hashing and EncryptedBlobStore for values\n that do not fit alongside the keys. At the moment, data can be\n overwritten, but not deleted.\n\n TODO:\n - Grow the dict by adding keysets\n - Migrating \"exciting\" keys to the primary keyset\n \"\"\"\n\n DEFAULT_BUCKET_SIZE = 5\n DEFAULT_DIGEST_SIZE = 16 # 128 bit hashes\n _UNUSED = '\\0U'\n _DELETED = '\\0D'\n\n MIN_KEY_BYTES = (48 - 16 - 7 - 1) # Magic number, smallest possible\n # key record size.\n\n DEFAULT_KEY_BYTES = (48 - 16 - 7 - 1) + 1*48 # 2 lines per record\n DEFAULT_DATA_BYTES = (48 - 16 - 7 - 1) + 8*48 # 9 lines ber record\n\n def __init__(self, base_fn, key,\n key_bytes=None,\n data_bytes=None,\n big_ratio=5,\n shard_size=100000, min_shards=1, shard_ratio=2.0,\n bucket_size=None,\n digest_size=None,\n overwrite=False,\n init_zeros=True,\n sparse=False):\n self._base_fn = base_fn\n self._key = key\n self._key_bytes = key_bytes or self.DEFAULT_KEY_BYTES\n self._data_bytes = (self.DEFAULT_DATA_BYTES\n if (data_bytes is None) else data_bytes)\n\n self._shard_ratio = shard_ratio # Growth ratio when expanding\n self._shard_size = max(int(shard_size), 1024)\n self._big_ratio = float(big_ratio)\n self._bucket_size = bucket_size or self.DEFAULT_BUCKET_SIZE\n self._digest_size = digest_size or self.DEFAULT_DIGEST_SIZE\n self._overwrite = overwrite\n self._sparse = sparse\n self._init_zeros = init_zeros\n\n self._lock = threading.RLock()\n\n self.load_factor = []\n self.writes = []\n self.reads = []\n self._keys = []\n while (os.path.exists(self._next_keyfile()[0]) or\n len(self._keys) < min_shards):\n self._load_next_keys()\n self.reset_counters()\n\n if big_ratio > 0:\n self._values = EncryptedBlobStore(self._base_fn, self._key,\n max_bytes=data_bytes,\n shard_size=shard_size,\n big_ratio=big_ratio,\n overwrite=overwrite)\n else:\n self._values = None\n\n def reset_counters(self):\n with self._lock:\n self.load_factor = [1.0 for keyset in self._keys]\n self.writes = [0 for keyset in self._keys]\n self.reads = [0 for keyset in self._keys]\n\n def _keyfile_size(self, kfi):\n return int(self._shard_size * (self._shard_ratio**kfi))\n\n def _keyfile_bytes(self, kfi):\n return self._key_bytes\n\n def _next_keyfile(self):\n pos = len(self._keys)\n return '%s-k-%s' % (self._base_fn, pos + 1), pos\n\n def _load_next_keys(self):\n with self._lock:\n kf, kfi = self._next_keyfile()\n kb = self._keyfile_bytes(kfi)\n ow = self._overwrite or not os.path.exists(kf)\n self._keys.append(EncryptedRecordStore(kf, self._key,\n max_bytes=kb, overwrite=ow, default_value=self._UNUSED))\n self.load_factor.append(0)\n self.writes.append(0)\n self.reads.append(0)\n if ow:\n kfs = self._keyfile_size(kfi)\n if self._sparse:\n self._keys[kfi][kfs - 1] = self._UNUSED\n elif self._init_zeros:\n self._keys[kfi].save_zeros(0, kfs)\n else:\n for i in range(0, kfs):\n self._keys[kfi][i] = self._UNUSED\n\n return kfi, self._keys[kfi]\n\n def _digest(self, key):\n digest = hashlib.sha256(self._key)\n if isinstance(key, unicode):\n key = key.encode('utf-8')\n elif not isinstance(key, str):\n key = str(key)\n digest.update(key)\n return digest.digest()[:self._digest_size]\n\n def _offset(self, digest):\n return struct.unpack(''):\n vpos = rdata[self._digest_size + 1:]\n vpos = struct.unpack('', vpos, len(value))])\n else:\n keyset[rpos] = ''.join([\n digest, struct.pack('',\n self._values.append(value),\n len(value))])\n else:\n keyset[rpos] = record_data\n return rpos\n elif on_fail is not None:\n return on_fail(self, kfi, keyset, pos, digest, value, records)\n else:\n return None\n\n def save_digest_record(self, digest, value, on_fail=None):\n pos = self._offset(digest)\n for kfi, keyset in enumerate(self._keys):\n rpos = self._try_save(kfi, keyset, pos, digest, value,\n on_fail=on_fail)\n if rpos is not None:\n self.writes[kfi] += 1\n return keyset, rpos\n\n # If we get this far, then we need to grow...\n if self._shard_ratio > 0:\n kfi, keyset = self._load_next_keys()\n rpos = self._try_save(kfi, keyset, pos, digest, value,\n on_fail=on_fail)\n if rpos is not None:\n self.writes[kfi] += 1\n return keyset, rpos\n\n raise KeyError('Save failed at %s' % pos)\n\n def save_record(self, key, value, on_fail=None):\n return self.save_digest_record(self._digest(key), value,\n on_fail=on_fail)\n\n def __setitem__(self, key, value):\n self.save_record(key, value)\n\n def __contains__(self, key):\n try:\n keyset, (rpos, rdata) = self.load_record(key)\n return True\n except (KeyError, ValueError, IndexError):\n return False\n\n def __delitem__(self, key):\n try:\n keyset, (rpos, rdata) = self.load_record(key)\n keyset[rpos] = self._DELETED\n except (KeyError, ValueError):\n pass\n\n def values(self):\n for keyset in self._keys:\n for val in keyset:\n try:\n yield self.rdata_value(val)\n except (KeyError, ValueError, IndexError):\n pass\n\n def rdata_digest(self, rdata):\n return rdata[:self._digest_size]\n\n def rdata_value(self, rdata):\n if rdata[self._digest_size] == '=':\n return rdata[self._digest_size + 1:]\n elif rdata[self._digest_size] == '>':\n pos, dlen = struct.unpack(' %s' % (i, ed.get(str(i))))\n done = time.time()\n print ('10k dict reads in %.2f (%.8f s/op)\\n -- reads=%s lf=%s'\n % (done - t0, (done - t0) / count, ed.reads, ed.load_factor))\n\n\n" }, { "path": "mailpile/crypto/state.py", "content": "from __future__ import print_function\n#. Common crypto state and structure\nimport copy\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nclass KeyLookupError(ValueError):\n def __init__(self, message, missing):\n ValueError.__init__(self, message)\n self.missing = missing\n\n\n# Crypto state is a strange beast, it needs to flow down and then\n# back up again - parts need to inherit a crypto context from their\n# container, but if something interesting is discovered that has to\n# be overridden.\n#\n# The discoveries then usually have to bubble up to influence the\n# overall state of that container and the message itself (mixed-*\n# states, etc).\n#\nclass CryptoInfo(dict):\n \"\"\"Base class for crypto-info classes\"\"\"\n KEYS = [\"protocol\", \"status\", \"description\"]\n STATUSES = [\"none\", \"mixed-error\", \"error\"]\n DEFAULTS = {\"status\": \"none\"}\n\n def __init__(self, parent=None, copy=None, bubbly=True):\n self.parent = parent\n self.bubbly = bubbly\n self.filename = None\n self.bubbles = []\n self._status = None\n if copy:\n self.update(copy)\n self._status = self.get(\"status\")\n elif parent:\n self.update(parent)\n self._status = self.get(\"status\")\n else:\n self.update(self.DEFAULTS)\n\n part_status = property(lambda self: (self._status or\n self.DEFAULTS[\"status\"]),\n lambda self, v: self._set_status(v))\n\n def _set_status(self, value):\n if value not in self.STATUSES:\n print('Bogus status for %s: %s' % (type(self), value))\n raise ValueError('Invalid status: %s' % value)\n self._status = value\n self.mix_bubbles()\n\n def __setitem__(self, item, value):\n if item not in self.KEYS:\n raise KeyError('Invalid key: %s' % item)\n if item == \"status\":\n if value not in self.STATUSES:\n print('Bogus status for %s: %s' % (type(self), value))\n raise ValueError('Invalid value for %s: %s' % (key, value))\n if self._status is None: # Capture initial value\n self._status = value\n dict.__setitem__(self, item, value)\n\n def _overwrite_with(self, ci):\n for k in self.keys():\n del self[k]\n self.update(ci)\n\n def bubble_up(self, parent=None):\n # Bubbling up adds this context to the list of contexts to be\n # evaluated for all parent states.\n if parent is not None and parent != self:\n self.parent = parent\n\n # Some contexts are neutral (pure MIME boilerplate) and do not\n # bubble up at all.\n if not self.bubbly:\n return\n\n parent = self.parent\n while parent is not None:\n parent.bubbles.append(self)\n parent = parent.parent\n\n def mix_bubbles(self):\n # Reset visible status to initial state\n self[\"status\"] = self.part_status\n # Mix in all the bubbly bubbles\n for bubble in self.bubbles:\n if bubble.bubbly:\n self._mix_in(bubble)\n\n def _mix_in(self, ci):\n \"\"\"\n This generates a mixed state for the message. The most exciting state\n is returned/explained, the status prefixed with \"mixed-\". How exciting\n states are, is determined by the order of the STATUSES attribute.\n\n This is lossy, but hopefully in a useful and non-harmful way.\n \"\"\"\n status = self[\"status\"]\n if self.STATUSES.index(status) <= self.STATUSES.index(ci.part_status):\n # ci is MORE or EQUALLY interesting\n mix = copy.copy(ci)\n if (self.bubbly and\n status != mix.part_status and\n not mix.part_status.startswith('mixed-')):\n mix[\"status\"] = \"mixed-%s\" % mix.part_status\n else:\n mix[\"status\"] = mix.part_status\n self._overwrite_with(mix)\n elif not status.startswith(\"mixed-\"):\n # ci is LESS interesting\n self[\"status\"] = 'mixed-%s' % status\n\n\nclass EncryptionInfo(CryptoInfo):\n \"\"\"Contains information about the encryption status of a MIME part\"\"\"\n KEYS = (CryptoInfo.KEYS + [\"have_keys\", \"missing_keys\", \"locked_keys\"])\n STATUSES = (CryptoInfo.STATUSES +\n [\"mixed-decrypted\", \"decrypted\",\n \"mixed-missingkey\", \"missingkey\",\n \"mixed-lockedkey\", \"lockedkey\"])\n\n\nclass SignatureInfo(CryptoInfo):\n \"\"\"Contains information about the signature status of a MIME part\"\"\"\n KEYS = (CryptoInfo.KEYS + [\"name\", \"email\", \"keyinfo\", \"timestamp\"])\n STATUSES = (CryptoInfo.STATUSES +\n [\"mixed-unknown\", \"unknown\",\n \"mixed-changed\", \"changed\", # TOFU; not the key we expected!\n \"mixed-unsigned\", \"unsigned\", # TOFU; should be signed!\n \"mixed-expired\", \"expired\",\n \"mixed-revoked\", \"revoked\",\n \"mixed-unverified\", \"unverified\",\n \"mixed-signed\", \"signed\", # TOFU; signature matches history\n \"mixed-verified\", \"verified\",\n \"mixed-invalid\", \"invalid\"])\n" }, { "path": "mailpile/crypto/streamer.py", "content": "from __future__ import print_function\n#\n# This is code to stream data to or from encrypted storage. If the invoking\n# code us correctly written, it should be able to work with data far in\n# excess of available RAM.\n#\n# The storage format \"Mailpile Encrypted Storage\" takes pains to be either\n# valid RFC2822 (for direct storage in IMAP servers) or a delimited format\n# similar to OpenPGP armour. Files must use one style or the other, not a\n# mixture of both.\n#\n# By default this code prefers to use AES-128-CTR. This cipher is malleable,\n# which means that data corruption is localized and does not affect the\n# rest of the file (unlike CBC, for example). In order to detect corruption\n# or attacks, a SHA256-based MAC can be calculated on the plaintext or\n# the ciphertext (or both). When verifying the plaintext, the sum is written\n# into the header of the file, when verifying the ciphertext the sum is\n# expected to be stored somewhere else.\n#\n##############################################################################\n# FIXME:\n#\n# The decryption routines here support \"MEP v1\" which used AES-256-CBC\n# and MD5 sums (not a proper MAC).\n#\n# Very few people have data in this format, it would be nice to just\n# delete all of that code once users have had time to migrate. But\n# for that to happen there needs to be a migration that finds old data\n# and re-encrypts it automatically. We don't have that yet!\n#\n##############################################################################\n#\nimport base64\nimport os\nimport hashlib\nimport sys\nimport re\nimport threading\nimport time\nimport traceback\nfrom datetime import datetime\nfrom tempfile import NamedTemporaryFile\n\nimport mailpile.platforms\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.safe_popen import Popen, PIPE\nfrom mailpile.util import CryptoLock, safe_remove, safe_assert\nfrom mailpile.util import sha512b64 as genkey\n\nfrom mailpile.crypto.aes_utils import getrandbits\nfrom mailpile.crypto.aes_utils import aes_ctr_encryptor, aes_ctr_decryptor\n\nPREFERRED_CIPHER = 'aes-128-ctr'\n\n\n# This is for backwards compatibility with v1 of our storage format; we have\n# since corrected our silliness and v2 uses a SHA256-based MAC.\nLEN_MD5_SUM = len(hashlib.md5('testing').hexdigest())\nMD5_SUM_FORMAT = 'md5sum: %s'\nMD5_SUM_PLACEHOLDER = MD5_SUM_FORMAT % ('0' * LEN_MD5_SUM)\nMD5_SUM_RE = re.compile('(?m)^' + MD5_SUM_FORMAT % (r'[^\\n]+',))\n\nLEN_SHA_256 = len(hashlib.sha256('testing').hexdigest())\nSHA_256_FORMAT = 'sha256: %s'\nSHA_256_PLACEHOLDER = SHA_256_FORMAT % ('0' * LEN_SHA_256)\nSHA_256_RE = re.compile('(?m)^' + SHA_256_FORMAT % (r'[^\\n]+',))\n\nBLANK_LINE_RE = re.compile('^\\s*$')\n\n\n# This gets populated with all the obsolete data we see during\n# decryption, the app can check for this to trigger migrations.\nPREFERRED_FORMAT = 'v2:%s' % PREFERRED_CIPHER\nDETECTED_OBSOLETE_FORMATS = set([])\n\nOPENSSL_COMMAND = mailpile.platforms.GetDefaultOpenSSLCommand\nOPENSSL_MD_ALG = \"md5\"\n\n# FIXME: Why does Windows require this? Move to mailpile.platforms when\n# we understand the underlying issue.\nif sys.platform.startswith(\"win\"):\n FILTER_MD5 = True\nelse:\n FILTER_MD5 = False\n\n\ndef mac_sha256(key, data):\n mac = hashlib.sha256(key or '')\n mac.update(data or '')\n return mac.hexdigest()\n\n\nclass IOFilter(threading.Thread):\n \"\"\"\n This class will wrap a filehandle and spawn a background thread to\n filter either the input or output.\n \"\"\"\n BLOCKSIZE = 16 * 1024\n\n def __init__(self, fd, callback,\n name=None, error_callback=None, blocksize=None):\n threading.Thread.__init__(self)\n self.callback = callback\n self.error_callback = error_callback\n self.exc_info = None\n\n self.blocksize = blocksize or self.BLOCKSIZE\n self.fd = fd\n self.writing = None\n self.reading_from = None\n self.writing_to = None\n\n self.exposed_fd = None\n self.my_pipe_fd = None\n\n pipe = os.pipe()\n self.pipe_reader = os.fdopen(pipe[0], 'rb', 0)\n self.pipe_writer = os.fdopen(pipe[1], 'wb', 0)\n\n self.info = 'Starting'\n self.aborting = False\n if name:\n self.name = name\n\n def __str__(self):\n return '%s: %s' % (threading.Thread.__str__(self), self.info)\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.close()\n\n def close(self):\n fd, self.exposed_fd = self.exposed_fd, None\n if fd is not None:\n try:\n fd.close()\n except OSError: # May already have been closed, that's fine\n pass\n\n if self.writing is False:\n self.aborting = 'Closed reader'\n\n self.join()\n\n def join(self, aborting=None):\n if aborting is not None:\n self.aborting = aborting\n return threading.Thread.join(self)\n\n def writer(self):\n if self.writing is None:\n self.writing = True\n\n self.reading_from = self.pipe_reader\n self.writing_to = self.fd\n self.my_pipe_fd = self.pipe_reader\n self.exposed_fd = self.pipe_writer\n\n self.start()\n return self.pipe_writer\n\n def reader(self):\n if self.writing is None:\n self.daemon = True\n self.writing = False\n\n self.reading_from = self.fd\n self.writing_to = self.pipe_writer\n self.my_pipe_fd = self.pipe_writer\n self.exposed_fd = self.pipe_reader\n\n self.start()\n return self.pipe_reader\n\n def _copy_loop(self):\n while not self.aborting:\n self.info = 'reading'\n data = self.reading_from.read(self.blocksize)\n if not self.aborting:\n if len(data) == 0:\n self.info = 'writing, EOF'\n self.writing_to.write(self.callback(None) or '')\n break\n else:\n self.info = 'writing'\n self.writing_to.write(self.callback(data))\n\n def run(self):\n okay = [AssertionError]\n if self.writing is False:\n # If we close early, we may get ValueErrors\n okay.append(ValueError)\n\n try:\n self.info = 'Starting: %s' % self.writing\n self._copy_loop()\n self.info += ', done'\n except tuple(okay):\n self.info += ', okay'\n pass\n except:\n self.info += ', failed'\n self.exc_info = sys.exc_info()\n traceback.print_exc()\n if self.error_callback:\n try:\n self.error_callback()\n except:\n pass\n finally:\n fd, self.my_pipe_fd = self.my_pipe_fd, None\n if fd is not None:\n self.info = 'Closing'\n fd.close()\n self.info = 'Dead'\n\n\nclass ReadLineIOFilter(IOFilter):\n \"\"\"\n This is a line-based IOFilter, which can stop when it sees a\n particular marker to hand off processing to others.\n \"\"\"\n def __init__(self, fd, callback,\n start_data=None, stop_check=None, **kwargs):\n self.stop_check = stop_check\n self.buffered = list(start_data)\n self.buf_bytes = sum(len(s) for s in start_data)\n IOFilter.__init__(self, fd, callback, **kwargs)\n\n def _maybe_flush(self, eof=False):\n if eof or (self.buf_bytes >= self.blocksize):\n i, self.info = self.info, 'flushing'\n self.writing_to.write(self.callback(''.join(self.buffered)))\n self.buffered = []\n self.buf_bytes = 0\n while eof:\n self.info = 'flushing EOF'\n data = self.callback(None) or ''\n self.writing_to.write(data)\n if not data:\n break\n self.info = i\n\n def _copy_loop(self):\n self.info = 'copying'\n for line in self.reading_from:\n if self.aborting: return\n\n self.buffered.append(line)\n if not re.match(BLANK_LINE_RE, line):\n # Don't count blank lines\n self.buf_bytes += len(line)\n self._maybe_flush()\n\n if self.aborting or (self.stop_check and self.stop_check(line)):\n break\n\n if not self.aborting:\n self._maybe_flush(eof=True)\n\n\nclass IOCoprocess(object):\n def __init__(self, command, fd, name=None, long_running=False):\n self.stderr = ''\n self._retval = None\n self._reading = False\n self.command = command\n self.name = name\n if command:\n try:\n self._proc, self._fd = self._popen(command, fd, long_running)\n except:\n print('Popen(%s, %s, %s)' % (command, fd, long_running))\n traceback.print_exc()\n print()\n raise\n else:\n self._proc, self._fd = None, fd\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.close()\n\n def close(self, *args):\n if self._retval is None:\n proc, fd, self._proc, self._fd = self._proc, self._fd, None, None\n if proc and fd:\n fd.close(*args)\n self.stderr = proc.stderr.read()\n\n # If we were reading from the process, not writing, then\n # closing our FD above may not be enough to terminate it,\n # and the following calls may hang. So kill kill kill.\n if self._reading:\n count = 0\n while proc.poll() is None:\n time.sleep(0.01)\n if count == 4:\n print('TERM => %s' % proc)\n proc.terminate()\n elif count > 9:\n print('KILL => %s' % proc)\n proc.kill()\n break\n count += 1\n\n self._retval = proc.wait()\n else:\n self._retval = 0\n return self._retval\n\n\nclass OutputCoprocess(IOCoprocess):\n \"\"\"\n This class will stream data to an external coprocess.\n \"\"\"\n def _popen(self, command, fd, long_running):\n proc = Popen(command, stdin=PIPE, stdout=fd, stderr=PIPE,\n bufsize=0, long_running=long_running)\n return proc, proc.stdin\n\n def _write_filter(self, data):\n return data\n\n def write(self, data, *args, **kwargs):\n return self._fd.write(self._write_filter(data), *args, **kwargs)\n\n def flush(self):\n return self._fd.flush()\n\n\nclass InputCoprocess(IOCoprocess):\n \"\"\"\n This class will stream data from an external coprocess.\n \"\"\"\n def _popen(self, command, fd, long_running):\n self._reading = True\n proc = Popen(command, stdin=fd, stdout=PIPE, stderr=PIPE,\n bufsize=0, long_running=long_running)\n return proc, proc.stdout\n\n def _read_filter(self, data):\n return data\n\n def __iter__(self, *args):\n return (self._read_filter(ln) for ln in self._fd.__iter__(*args))\n\n def readline(self, *args):\n return self._read_filter(self._fd.readline(*args))\n\n def readlines(self, *args):\n return [self._read_filter(ln) for ln in self._fd.readlines(*args)]\n\n def read(self, *args):\n return self._read_filter(self._fd.read(*args))\n\n\nclass ChecksummingStreamer(OutputCoprocess):\n \"\"\"\n This checksums and streams data to a named temporary file on disk, which\n can then be read back or linked to a final location.\n \"\"\"\n FILTER_BLOCKSIZE = None\n\n def __init__(self, dir=None, name=None, long_running=False,\n use_filter=FILTER_MD5):\n self.tempfile, self.temppath = self._mk_tempfile_and_path(dir)\n self.name = name\n\n self.outer_sha256 = None\n if use_filter:\n self.outer_sha = hashlib.sha256()\n self.shafilter = IOFilter(self.tempfile, self._sha256_callback,\n name='%s/sha256' % (self.name or 'css'),\n blocksize=self.FILTER_BLOCKSIZE)\n self.fd = self.shafilter.writer()\n else:\n self.outer_sha = None\n self.fd = self.tempfile\n\n self.saved = False\n self.finished = False\n try:\n self._write_preamble()\n OutputCoprocess.__init__(self, self._mk_command(), self.fd,\n name=self.name,\n long_running=long_running)\n except:\n try:\n self.fd.close()\n if self.outer_sha is not None:\n self.shafilter.close()\n self.tempfile.close()\n safe_remove(self.temppath)\n except (IOError, OSError):\n pass\n raise\n\n def _mk_tempfile_and_path(self, _dir):\n ntf = NamedTemporaryFile(dir=_dir, delete=False)\n return ntf, ntf.name\n\n def _mk_command(self):\n return None\n\n def finish(self):\n fin, self.finished = self.finished, True\n if fin:\n return\n # Stop sending output to our coprocess, wait for it to finish\n OutputCoprocess.close(self)\n\n # Write postamble (the shafilter), close that too\n self.tempfile.seek(0, 2)\n self._write_postamble()\n\n if self.outer_sha is None:\n # If we weren't doing the SHA256 on the fly, do it now.\n self.calculate_outer_sha256()\n else:\n # Otherwise, close our coprocess to trigger the calculation\n self.fd.close()\n self.shafilter.close()\n\n # Reset our tempfile to the beginning for reading\n self.tempfile.seek(0, 0)\n\n def close(self):\n self.finish()\n self.tempfile.close()\n\n def save(self, filename, finish=True, mode='wb'):\n if finish:\n self.finish()\n\n # If no filename, return contents to caller\n if filename is None:\n if not self.saved:\n safe_remove(self.temppath)\n self.saved = True\n self.tempfile.seek(0, 0)\n return self.tempfile.read()\n\n # 1st save just renames the tempfile\n exists = os.path.exists(filename)\n if (not self.saved and\n (('a' not in mode) or not exists)):\n try:\n if exists:\n os.remove(filename)\n os.rename(self.temppath, filename)\n self.saved = True\n return\n except (OSError, IOError):\n pass\n\n # 2nd save (or append to existing) creates a copy\n with open(filename, mode) as out:\n self.save_copy(out)\n if not self.saved:\n safe_remove(self.temppath)\n self.saved = True\n\n def calculate_outer_sha256(self):\n self.tempfile.seek(0, 0)\n data = self.tempfile.read(4096)\n self.outer_sha = outer_sha = hashlib.sha256()\n while data != '':\n # We calculate the MD5 sum as if the data used the CRLF linefeed\n # convention, whether it's actually using that or not.\n outer_sha.update(data.replace('\\r', '').replace('\\n', '\\r\\n'))\n data = self.tempfile.read(4096)\n self.outer_sha256 = outer_sha.hexdigest()\n\n def outer_mac_sha256(self):\n # Hm, we have no key, so this is a bit pointless\n return mac_sha256('', self.outer_sha.digest())\n\n def save_copy(self, ofd):\n self.tempfile.seek(0, 0)\n data = self.tempfile.read(4096)\n while data != '':\n ofd.write(data)\n data = self.tempfile.read(4096)\n\n def _sha256_callback(self, data):\n if data is None:\n # EOF...\n self.outer_sha256 = self.outer_sha.hexdigest()\n return ''\n else:\n # We calculate the MD5 sum as if the data used the CRLF linefeed\n # convention, whether it's actually using that or not.\n self.outer_sha.update(data.replace('\\r', '').replace('\\n', '\\r\\n'))\n return data\n\n def _write_preamble(self):\n pass\n\n def _write_postamble(self):\n pass\n\n\nclass EncryptingDelimitedStreamer(ChecksummingStreamer):\n \"\"\"\n This class creates a coprocess for encrypting data. The data will\n be streamed to a named temporary file on disk, which can then be\n read back or linked to a final location.\n \"\"\"\n BEGIN_DATA = \"-----BEGIN MAILPILE ENCRYPTED DATA-----\\n\"\n EXTRA_HEADERS = \"X-Mailpile-Encrypted-Data: v2\\n\"\n EXTRA_DATA = {}\n END_DATA = \"-----END MAILPILE ENCRYPTED DATA-----\\n\"\n\n PREFERRED_CIPHER = None\n FILTER_BLOCKSIZE = 19 * 3 * 16 # Make AES and Base64 happy\n\n def __init__(self, key,\n dir=None, cipher=None, name=None, header_data=None,\n long_running=False, use_filter=FILTER_MD5):\n self.cipher = cipher or self.PREFERRED_CIPHER or PREFERRED_CIPHER\n self.nonce, self.key = self._nonce_and_mutated_key(key)\n self.header_data = (header_data if header_data is not None\n else self.EXTRA_DATA)\n\n self.encode_buffer = ''\n if self.cipher == 'aes-128-ctr':\n self.encryptor = aes_ctr_encryptor(self.key, self.nonce)\n self.encoder = base64.encodestring\n self.encode_batches = self.FILTER_BLOCKSIZE\n elif self.cipher == 'none':\n self.encryptor = lambda d: d\n self.encoder = base64.encodestring\n self.encode_batches = self.FILTER_BLOCKSIZE\n elif self.cipher == 'broken':\n self.encoder = self.encryptor = lambda d: d\n self.encode_batches = None\n else:\n self.encoder = self.encryptor = None\n self.encode_batches = None\n\n ChecksummingStreamer.__init__(self, dir=dir, name=name,\n long_running=long_running,\n use_filter=use_filter)\n\n # These are necessary for two reasons:\n #\n # 1. Prevent the public checksum from revealing what was encrypted.\n # 2. When using malleable encryption modes (such as CTR), make it\n # infeasable for attackers to generate a checksum that validates\n # modified data.\n #\n # As this is for data at rest, we are not particularly concerned with\n # oracle attacks. If an attacker has access to your mailpile on disk\n # while it is in use (so they can inject new messages), there are\n # other attacks which are both easier and more severe.\n #\n self.inner_sha256 = None\n self.inner_sha = hashlib.sha256()\n self.inner_sha.update(self.key)\n self.inner_sha.update(self.nonce or '')\n\n self._send_key()\n\n def _write_filter(self, data):\n if data:\n self.inner_sha.update(data)\n if self.encryptor and (data or self.encode_buffer):\n if self.encode_batches:\n eof = not data\n if data: self.encode_buffer += data\n data = ''\n for i in (512, 128, 8, 1):\n batch = i * self.encode_batches\n while eof or (len(self.encode_buffer) >= batch):\n d = self.encode_buffer[:batch]\n b = self.encode_buffer[batch:]\n self.encode_buffer = b\n data += self.encoder(self.encryptor(d))\n eof = False\n return data\n\n def _sha256_callback(self, data):\n return ChecksummingStreamer._sha256_callback(self, data)\n\n def outer_mac_sha256(self):\n return mac_sha256(self.key or '', self.outer_sha.digest())\n\n def write_pad_and_flush(self, data, pad=' '):\n if self.encryptor and (data or self.encode_buffer):\n if self.encode_batches:\n remainder = len(self.encode_buffer) + len(data)\n remainder %= self.encode_batches\n padding = self.encode_batches - remainder\n data += (pad * padding)\n self.write(data)\n self.flush()\n\n def finish(self, *args, **kwargs):\n if not self.finished:\n while self.encode_buffer:\n self.write('')\n rv = ChecksummingStreamer.finish(self, *args, **kwargs)\n self._write_inner_sha256()\n return rv\n else:\n return ChecksummingStreamer.finish(self, *args, **kwargs)\n\n def _write_inner_sha256(self):\n if not self.inner_sha256:\n self.inner_sha256 = mac_sha256(self.key, self.inner_sha.digest())\n pos = self.tempfile.tell()\n self.tempfile.seek(0, 0)\n old_data = self.tempfile.read(4096)\n\n sha_256_header = SHA_256_FORMAT % (self.inner_sha256, )\n new_data = re.sub(SHA_256_RE, sha_256_header, old_data)\n if old_data != new_data:\n self.tempfile.seek(0, 0)\n self.tempfile.write(new_data)\n\n self.tempfile.seek(pos, 0)\n\n def _nonce_and_mutated_key(self, key):\n # This generates a nonce which may be used as a salt, IV, or\n # counter-prefix depending the algorithm and mode in use. We\n # also use it to derive a mutated key for each message, thus\n # reducing the risks of the (key, iv) pairs ever repeating even\n # if a mistake is made somewhere else.\n nonce = '%32.32x' % getrandbits(32 * 4)\n return nonce, genkey(key, nonce)[:32].strip()\n\n def _send_key(self):\n # We talk directly to the underlying FD, to avoid corrupting the\n # inner MD5 sum (calculated using the _write_filter() above).\n if not self.encryptor:\n self._fd.write('%s\\n' % self.key)\n\n def _mk_command(self):\n if self.encryptor:\n return None\n return [OPENSSL_COMMAND(), \"enc\", \"-e\", \"-a\", \"-%s\" % self.cipher,\n \"-pass\", \"stdin\", \"-bufsize\", \"0\", \"-md\", OPENSSL_MD_ALG]\n\n def _write_preamble(self):\n self.fd.write(self.BEGIN_DATA)\n self.fd.write('cipher: %s\\n' % self.cipher)\n if self.nonce:\n self.fd.write('nonce: %s\\n' % self.nonce)\n self.fd.write(SHA_256_PLACEHOLDER + '\\n')\n if self.EXTRA_HEADERS:\n self.fd.write(self.EXTRA_HEADERS % self.header_data)\n self.fd.write('\\n')\n self.fd.flush()\n\n def _write_postamble(self):\n if self.END_DATA:\n self.fd.write('\\n')\n self.fd.write(self.END_DATA)\n self.fd.flush()\n\n\nclass EncryptingUndelimitedStreamer(EncryptingDelimitedStreamer):\n \"\"\"\n This class creates a coprocess for encrypting data. The data will\n be streamed to a named temporary file on disk, which can then be\n read back or linked to a final location.\n \"\"\"\n BEGIN_DATA = \"X-Mailpile-Encrypted-Data: v2\\n\"\n EXTRA_HEADERS = (\"From: Mailpile \\n\"\n \"Subject: %(subject)s\\n\")\n EXTRA_DATA = {'subject': 'Mailpile encrypted data'}\n END_DATA = \"\"\n\n\ndef EncryptingStreamer(*args, **kwargs):\n delimited = kwargs.get('delimited', False)\n if 'delimited' in kwargs:\n del kwargs['delimited']\n if delimited:\n return EncryptingDelimitedStreamer(*args, **kwargs)\n else:\n return EncryptingUndelimitedStreamer(*args, **kwargs)\n\n\nclass DecryptingStreamer(InputCoprocess):\n \"\"\"\n This class creates a coprocess for decrypting data.\n \"\"\"\n BEGIN_PGP = \"-----BEGIN PGP MESSAGE-----\"\n END_PGP = \"-----END PGP MESSAGE-----\"\n BEGIN_MED = \"-----BEGIN MAILPILE ENCRYPTED DATA-----\"\n BEGIN_MED2 = \"X-Mailpile-Encrypted-Data: \"\n END_MED = \"-----END MAILPILE ENCRYPTED DATA-----\"\n PREFERRED_CIPHER = None\n\n STATE_BEGIN = 0\n STATE_HEADER = 1\n STATE_DATA = 2\n STATE_ONLY_DATA = 3\n STATE_END = 4\n STATE_RAW_DATA = 5\n STATE_PGP_DATA = 6\n STATE_ERROR = -1\n\n @classmethod\n def StartEncrypted(cls, line):\n return (line.startswith(cls.BEGIN_MED) or\n line.startswith(cls.BEGIN_MED2) or\n line.startswith(cls.BEGIN_PGP))\n\n @classmethod\n def EndEncrypted(cls, line):\n return (line.startswith(cls.END_MED) or\n line.startswith(cls.END_PGP))\n\n def __init__(self, fd,\n mep_key=None, gpg_pass=None, sha256=None, cipher=None,\n name=None, long_running=False, gpgi=None, md_alg=None):\n self.expected_outer_sha256 = sha256\n self.expected_inner_sha256 = None\n self.expected_inner_md5sum = None\n self.name = name\n self.outer_sha = hashlib.sha256()\n self.inner_sha = hashlib.sha256()\n self.inner_md5 = hashlib.md5()\n self.cipher = self.PREFERRED_CIPHER or PREFERRED_CIPHER\n self.md_alg = md_alg or OPENSSL_MD_ALG\n self.state = self.STATE_BEGIN\n self.buffered = ''\n self.mep_version = None\n self.mep_mutated = None\n self.mep_key = mep_key\n self.gpg_pass = gpg_pass\n self.gpgi = gpgi\n self.decryptor = None\n self.decoder = None\n self.decoder_data_bytes = 0 # Not counting white-space\n\n # Start reading our data...\n self.startup_lock = CryptoLock()\n self.startup_lock.acquire()\n self.data_filter = self._mk_data_filter(fd, self._read_data,\n self.startup_lock.release)\n self.read_fd = self.data_filter.reader()\n try:\n # Once the header has been processed (_read_data() will release\n # the lock), fork out our coprocess.\n self.startup_lock.acquire()\n InputCoprocess.__init__(self, self._mk_command(), self.read_fd,\n name=name, long_running=long_running)\n except:\n try:\n self.data_filter.join(aborting=True)\n self.data_filter.close()\n except (IOError, OSError):\n pass\n raise\n finally:\n self.startup_lock.release()\n self.startup_lock = None\n\n def _read_filter(self, data):\n if data:\n if self.expected_inner_sha256:\n self.inner_sha.update(data)\n if self.expected_inner_md5sum:\n self.inner_md5.update(data)\n return data\n\n def close(self):\n self.read_fd.close()\n self.data_filter.join()\n return InputCoprocess.close(self)\n\n def verify(self, testing=False, _raise=None):\n if self.close() != 0:\n if testing:\n print('Close returned nonzero')\n if _raise:\n raise _raise('Non-zero exit code from coprocess')\n return False\n\n if self.expected_inner_sha256:\n mac = mac_sha256(self.mep_mutated, self.inner_sha.digest())\n if self.expected_inner_sha256 != mac:\n if testing:\n print('Inner %s != %s' % (self.expected_inner_sha256, mac))\n if _raise:\n raise _raise('Invalid inner SHA256')\n return False\n elif self.expected_inner_md5sum:\n if self.expected_inner_md5sum != self.inner_md5.hexdigest():\n if testing:\n print('Inner %s != %s' % (self.expected_inner_md5sum,\n self.inner_md5.hexdigest()))\n if _raise:\n raise _raise('Invalid inner MD5 sum')\n return False\n elif testing and not self.expected_inner_md5sum:\n print('No inner MD5 sum or SHA256 expected')\n\n if self.expected_outer_sha256:\n mac = mac_sha256(self.mep_mutated, self.outer_sha.digest())\n if self.expected_outer_sha256 != mac:\n if testing:\n print('Outer %s != %s' % (self.expected_outer_sha256, mac))\n if _raise:\n raise _raise('Invalid outer SHA256')\n return False\n elif testing and not self.expected_outer_sha256:\n print('No outer SHA256 expected')\n return True\n\n def _mk_data_filter(self, fd, cb, ecb):\n return IOFilter(fd, cb, error_callback=ecb,\n name='%s/filter' % (self.name or 'ds'))\n\n def _read_data(self, data):\n def process(data):\n if self.decryptor is not None:\n eof = not data\n if self.decoder_data_bytes and data:\n self.buffered += ''.join([c for c in data if c not in\n (' ', '\\t', '\\r', '\\n')])\n else:\n self.buffered += (data or '')\n data = ''\n for i in (256, 64, 8, 1):\n batch = max(1, i * self.decoder_data_bytes)\n while eof or (len(self.buffered) >= batch):\n if self.decoder_data_bytes:\n d = self.buffered[:batch]\n b = self.buffered[batch:]\n self.buffered = b\n else:\n d, self.buffered = self.buffered, ''\n try:\n data += self.decryptor(self.decoder(d))\n eof = False\n except TypeError:\n raise IOError('%s: Bad data, failed to decode'\n % self.name)\n return (data or '')\n\n if data is None:\n # EOF!\n if self.state in (self.STATE_BEGIN, self.STATE_HEADER):\n self.state = self.STATE_RAW_DATA\n self.startup_lock.release()\n data, self.buffered = self.buffered, ''\n return process(data) + process(None)\n return process(None)\n\n if self.expected_outer_sha256:\n # The outer MD5 sum is calculated over all data, but with any\n # CRLF sequences normalized to only LF and the sha256 header\n # itself replaced with a placeholder.\n if self.state in (self.STATE_BEGIN, self.STATE_HEADER):\n sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER,\n re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data))\n else:\n sum_data = data\n sum_data = sum_data.replace('\\r', '').replace('\\n', '\\r\\n')\n self.outer_sha.update(sum_data)\n\n if self.state in (\n self.STATE_RAW_DATA, self.STATE_PGP_DATA, self.STATE_ONLY_DATA):\n return process(data)\n\n if self.state == self.STATE_BEGIN:\n self.buffered += data\n if (len(self.buffered) >= len(self.BEGIN_PGP)\n and self.buffered.startswith(self.BEGIN_PGP)):\n self.state = self.STATE_PGP_DATA\n if self.gpg_pass:\n self.gpg_pass.seek(0, 0)\n passphrase, c = [], self.gpg_pass.read(1)\n while c != '':\n passphrase.append(c)\n c = self.gpg_pass.read(1)\n self.startup_lock.release()\n return ''.join(passphrase + ['\\n', self.buffered])\n else:\n self.startup_lock.release()\n return self.buffered\n\n # Note: The max() check is OK, because both formats add more\n # data which covers the difference.\n if len(self.buffered) >= max(len(self.BEGIN_MED),\n len(self.BEGIN_MED2)):\n if not (self.buffered.startswith(self.BEGIN_MED) or\n self.buffered.startswith(self.BEGIN_MED2)):\n self.state = self.STATE_RAW_DATA\n self.startup_lock.release()\n return self.buffered\n if '\\r\\n\\r\\n' in self.buffered:\n header, data = self.buffered.split('\\r\\n\\r\\n', 1)\n headlines = header.strip().split('\\r\\n')\n self.state = self.STATE_HEADER\n elif '\\n\\n' in self.buffered:\n header, data = self.buffered.split('\\n\\n', 1)\n headlines = header.strip().split('\\n')\n self.state = self.STATE_HEADER\n else:\n return ''\n else:\n return ''\n\n if self.state == self.STATE_HEADER:\n # State: header and data have been set, header is complete.\n self.buffered = ''\n\n headers = dict([l.split(': ', 1) for l in headlines if ': ' in l])\n nonce = headers.get('nonce', '')\n self.mep_mutated = self._mutate_key(self.mep_key, nonce)\n self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1')\n self.cipher = headers.get('cipher', self.cipher)\n\n data_fmt = '%s:%s' % (self.mep_version, self.cipher)\n if data_fmt != PREFERRED_FORMAT:\n DETECTED_OBSOLETE_FORMATS.add(data_fmt)\n\n eim = self.expected_inner_md5sum = headers.get('md5sum')\n if eim and eim == ('0' * LEN_MD5_SUM):\n self.expected_inner_md5sum = None\n if self.expected_inner_md5sum:\n self.inner_md5.update(self.mep_mutated)\n self.inner_md5.update(nonce)\n\n eis = self.expected_inner_sha256 = headers.get('sha256')\n if eis and eis == ('0' * LEN_SHA_256):\n self.expected_inner_sha256 = None\n if self.expected_inner_sha256:\n self.inner_sha.update(self.mep_mutated)\n self.inner_sha.update(nonce)\n\n if self.buffered.startswith(self.BEGIN_MED2):\n self.state = self.STATE_ONLY_DATA\n else:\n self.state = self.STATE_DATA\n\n if self.cipher == 'aes-128-ctr':\n self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce)\n self.decoder = base64.b64decode\n # Decode data in chunks this big (multiple of 4 and 16);\n # guarantees workable chunks for both AES-CTR and base64.\n self.decoder_data_bytes = 32 * 1024\n elif self.cipher == 'none':\n self.decryptor = lambda d: d\n self.decoder = base64.b64decode\n self.decoder_data_bytes = 32 * 1024\n elif self.cipher == 'broken':\n self.decryptor = lambda d: d\n self.decoder = lambda d: d\n self.expected_inner_md5sum = None\n self.expected_inner_sha256 = None\n else:\n self.decryptor = None\n data = '\\n'.join((self.mep_mutated, data))\n\n self.startup_lock.release()\n\n if self.state == self.STATE_ONLY_DATA:\n return process(data)\n\n if self.state == self.STATE_DATA:\n for delim in (self.END_MED, self.END_PGP):\n if delim in data:\n for pf in ('\\r\\n', '\\n', ''):\n if pf + delim in data:\n data = data.split(pf + delim, 1)[0]\n self.state = self.STATE_END\n return process(data)\n return process(data)\n\n # Error, end and unknown states...\n return ''\n\n def _mutate_key(self, key, nonce):\n return genkey(key or '', nonce)[:32].strip()\n\n def _mk_command(self):\n if self.state == self.STATE_RAW_DATA:\n return None\n elif self.decryptor is not None:\n return None\n elif self.state == self.STATE_PGP_DATA:\n safe_assert(self.gpgi is not None)\n if self.gpg_pass:\n return self.gpgi.common_args(will_send_passphrase=True)\n else:\n return self.gpgi.common_args()\n return [OPENSSL_COMMAND(), \"enc\", \"-d\", \"-a\", \"-%s\" % self.cipher,\n \"-pass\", \"stdin\", \"-md\", self.md_alg]\n\n\nclass PartialDecryptingStreamer(DecryptingStreamer):\n def __init__(self, start_data, *args, **kwargs):\n self.start_data = start_data\n DecryptingStreamer.__init__(self, *args, **kwargs)\n\n def _mk_data_filter(self, fd, cb, ecb):\n return ReadLineIOFilter(fd, cb,\n start_data=self.start_data,\n stop_check=self.EndEncrypted,\n error_callback=ecb,\n name='%s/rlfilter' % (self.name or 'ds'))\n\n\nif __name__ == \"__main__\":\n import random # See! Not in the main module!\n import StringIO\n\n def _assert(val, want=True, msg='assert'):\n if isinstance(want, bool):\n if (not val) == (not want):\n want = val\n if val != want:\n raise AssertionError('%s(%s==%s)' % (msg, val, want))\n\n LEGACY_TEST_KEY = 'test key'\n LEGACY_PLAINTEXT = 'Hello world! This is great!\\nHooray, lalalalla!\\n'\n LEGACY_TEST_1 = \"\"\"\\\nX-Mailpile-Encrypted-Data: v1\ncipher: aes-256-cbc\nnonce: SEefbOfc9UQmZeWWGWQMrb0n6czXY2Uv\nmd5sum: b07d3ed58b79a69ab5496cffcab5d878\nFrom: Mailpile \nSubject: Mailpile encrypted data\n\nU2FsdGVkX18zVuMErdegtGziWDLhSvNRb7YRRxmYKMmygI1H3bp+mXffToii6lGB\nZ7Vlo78g20D8NAO6dpJfmA==\n\"\"\"\n LEGACY_TEST_2 = \"\"\"\\\n-----BEGIN MAILPILE ENCRYPTED DATA-----\ncipher: aes-256-cbc\nnonce: SB+fmmM72oFpf/FO4wnaHhFBvhgzpbwW\nmd5sum: 90dfb2850da49c8a6027415521dadb3c\n\nU2FsdGVkX19U8G7SKp8QygUusdHZThlrLcI04+jZ9U5kwfsw7bJJ2721dwgIpCUh\n3wpQjsYtFF2dcKBjrG7xyw==\n\n-----END MAILPILE ENCRYPTED DATA-----\n\"\"\"\n\n # Do this before checking for fd leaks, as it may open up /dev/urandom\n # and keep it open.\n b = getrandbits(128)\n\n # Create a pipe, this tells us which FDs are available next\n fdpair1 = os.pipe()\n for fd in fdpair1:\n os.close(fd)\n def fdcheck(where):\n fdpair2 = os.pipe()\n try:\n for fd in fdpair2:\n if fd not in fdpair1:\n print('Probably have an FD leak at %s!' % where)\n print('Verify with: lsof -g %s' % os.getpid())\n import time\n time.sleep(900)\n return False\n return True\n finally:\n for fd in fdpair2:\n os.close(fd)\n\n bc = [0]\n def counter(data):\n bc[0] += len(data or '')\n return (data or '')\n\n # Cleanup...\n try:\n os.unlink('/tmp/iofilter.tmp')\n except OSError:\n pass\n\n print('Test the IOFilter in write mode')\n with open('/tmp/iofilter.tmp', 'w') as bfd:\n with IOFilter(bfd, counter) as iof:\n iof.writer().write('Hello world!')\n with open('/tmp/iofilter.tmp', 'r') as iof:\n assert(iof.read() == 'Hello world!')\n _assert(bc[0], 12)\n _assert(fdcheck('IOFilter in write mode'))\n\n print('Test the IOFilter in read mode')\n bc[0] = 0\n with open('/tmp/iofilter.tmp', 'r') as bfd:\n with IOFilter(bfd, counter) as iof:\n data = iof.reader().read()\n _assert(data, 'Hello world!')\n _assert(bc[0], 12)\n _assert(fdcheck('IOFilter in read mode'))\n\n print('Test the IOFilter in incomplete read mode')\n bc[0] = 0\n with open('/dev/urandom', 'r') as bfd:\n with IOFilter(bfd, counter) as iof:\n data = iof.reader().read(4096)\n _assert(bc[0] >= 4096, msg='%s >= 4096' % bc[0])\n _assert(len(data) == 4096)\n _assert(fdcheck('IOFilter in incomplete read mode'))\n\n print('Test the ReadLineIOFilter in incomplete read mode')\n bc[0], daemonlogline = 0, ''\n with open('/etc/passwd', 'r') as bfd:\n with IOFilter(bfd, counter) as iof:\n for line in iof.reader():\n if 'daemon' in line:\n daemonlogline = line\n break\n _assert(bc[0] > 80, msg='%s > 80' % bc[0])\n _assert('daemon' in daemonlogline, msg='daemon in %s' % daemonlogline)\n _assert(fdcheck('ReadLineIOFilter in incomplete read mode'))\n\n print('Null decryption test, sha256 verification only')\n outer_mac_sha256 = '7982970534e089b839957b7e174725ce1878731ed6d700766e59cb16f1c25e27'\n with open('/tmp/iofilter.tmp', 'rb') as bfd:\n with DecryptingStreamer(bfd,\n mep_key='test key',\n sha256=outer_mac_sha256\n ) as ds:\n _assert('Hello world!', ds.read())\n _assert(ds.verify(testing=True))\n _assert(fdcheck('Decrypting test, sha256 verification'))\n\n print('Legacy (MEP v1) decryption test')\n for legacy in (LEGACY_TEST_1, LEGACY_TEST_2):\n lfd = StringIO.StringIO(legacy)\n with PartialDecryptingStreamer([], lfd,\n mep_key=LEGACY_TEST_KEY) as ds:\n plaintext = ''\n d = ds.read(1)\n while len(d) > 0:\n plaintext += d\n d = ds.read(random.randint(10, max(11, len(legacy))))\n try:\n _assert(plaintext, LEGACY_PLAINTEXT)\n _assert(ds.verify(testing=True))\n except AssertionError:\n print('command=%s' % ds.command)\n print('stderr=%s' % ds.stderr)\n print('key=%s [%s]\\n%s' % (LEGACY_TEST_KEY, ds.mep_mutated, legacy))\n raise\n\n for cipher in ('none', 'broken', 'aes-128-ctr', 'aes-256-cbc'):\n for filter_sha256 in (True, False):\n for delim in (True, False):\n print(('Encryption test, cipher=%s, delim=%s, filter_sha256=%s'\n ) % (cipher, delim, filter_sha256))\n\n fn = '/tmp/enc-%s-%s-%s.tmp' % (cipher, delim, filter_sha256)\n with open(fn, 'wb') as fd:\n fd.write('junk') # Make sure overwriting works\n\n t0 = time.time()\n mul = 123400\n data = 'Hello world! This is great!\\nHooray, lalalalla!\\n' * mul\n parts = [(data, 'wb')]\n if delim:\n for i in (2, 5, 10, mul // 10, mul):\n more = 'part two, yeaaaah\\n' * (mul // i)\n parts.append((more, 'ab'))\n data += more\n encrypted = []\n for part, mode in parts:\n with EncryptingStreamer('test key', dir='/tmp',\n delimited=delim,\n cipher=cipher,\n use_filter=filter_sha256) as es:\n d = part\n while d:\n i = min(len(d), random.randint(10, max(11, len(d))))\n es.write(d[:i])\n d = d[i:]\n es.finish()\n es.save(fn, mode=mode)\n encrypted.append(es.outer_mac_sha256())\n _assert(fdcheck('Encrypted data, delimited=%s' % delim))\n\n t1 = time.time()\n print('Decryption test, delim=%s' % delim)\n with open(fn, 'rb') as bfd:\n new_data = ''\n for ms in encrypted:\n if delim:\n ds = PartialDecryptingStreamer(\n [], bfd, mep_key='test key', sha256=ms)\n else:\n ds = DecryptingStreamer(\n bfd, mep_key='test key', sha256=ms)\n with ds:\n d = ds.read(9999)\n while d:\n new_data += d\n d = ds.read(random.randint(10, 102400))\n _assert(ds.verify(testing=True))\n try:\n _assert(data, new_data)\n except:\n print('OLD %d bytes vs. NEW %d bytes: \\n%s\\n' % (\n len(data), len(new_data), new_data[-100:]))\n raise\n _assert(fdcheck('Decrypting test, delimited=%s' % delim))\n t2 = time.time()\n print (' => Elapsed: %.3fs + %.3fs = %.3fs (%.2f MB/s)'\n % (t1-t0, t2-t1, t2-t0, len(new_data)/(1024*1024*(t2-t0))))\n\n # Cleanup\n os.unlink(fn)\n print()\n\n _assert(len(DETECTED_OBSOLETE_FORMATS) > 0)\n print('Obsolete formats detected: %s' % DETECTED_OBSOLETE_FORMATS)\n\n os.unlink('/tmp/iofilter.tmp')\n _assert(fdcheck('All done'))\n" }, { "path": "mailpile/crypto/tor.py", "content": "import copy\nimport socket\nimport subprocess\nimport threading\nimport time\nimport traceback\nimport re\nimport sys\nimport os\n\nimport stem.process\nimport stem.control\n\nimport mailpile.util\nfrom mailpile.eventlog import Event\nfrom mailpile.platforms import RandomListeningPort, GetDefaultTorPath\nfrom mailpile.safe_popen import PresetSafePopenArgs, MakePopenSafe\nfrom mailpile.util import okay_random\n\nif 'pythonw' in sys.executable:\n debug_target = open( os.devnull, 'w' )\nelse:\n debug_target = sys.stdout\n\ndef debug( text ):\n debug_target.write( text + '\\n' )\n debug_target.flush()\n\n# Version check for STEM >= 1.4\nassert(int(stem.__version__[0]) > 1 or\n (int(stem.__version__[0]) == 1 and int(stem.__version__[2]) >= 4))\n\n\nclass Tor(threading.Thread):\n _instance = None\n def __new__(cls, *args, **kwargs):\n if not cls._instance:\n cls._instance = super(Tor, cls).__new__(cls, *args, **kwargs)\n return cls._instance\n\n def __init__(self, session=None, config=None,\n socks_port=None, control_port=None, tor_binary=None,\n callbacks=None):\n threading.Thread.__init__(self)\n self.session = session\n self.config = config or (session.config if session else None)\n self.callbacks = callbacks\n\n if self.config is None:\n self.socks_port = None\n self.control_port = None\n self.control_password = okay_random(32)\n self.tor_binary = tor_binary\n else:\n self.socks_port = self.config.sys.tor.socks_port\n self.control_port = self.config.sys.tor.ctrl_port\n self.control_password = self.config.sys.tor.ctrl_auth\n self.tor_binary = tor_binary or self.config.sys.tor.binary or None\n\n if socks_port is not None:\n self.socks_port = socks_port\n if control_port is not None:\n self.control_port = control_port\n\n self.event = Event(source=self, flags=Event.INCOMPLETE, data={})\n self.lock = threading.Lock()\n self.tor_process = None\n self.tor_controller = None\n self.hidden_services = {}\n self.keep_looping = True\n self.started = False\n\n def run(self):\n starts = 0\n while self.keep_looping and not mailpile.util.QUITTING:\n starts += 1\n try:\n self._run_once()\n except OSError:\n pass\n for i in range(0, 5 * min(60, starts)):\n if mailpile.util.QUITTING: break\n time.sleep(0.2)\n\n def _run_once(self):\n try:\n random_ports = RandomListeningPort(count=2)\n with self.lock:\n self.event.flags = Event.INCOMPLETE\n self.tor_process = 'starting up'\n self.tor_controller = None\n self.started = True\n\n if not self.socks_port:\n self.socks_port = random_ports[0]\n if not self.control_port:\n self.control_port = random_ports[1]\n if self.tor_binary is None:\n self.tor_binary = GetDefaultTorPath()\n\n tor_process_config = {\n 'SocksPort': str(self.socks_port),\n 'ControlPort': str(self.control_port),\n 'HashedControlPassword': self._hashed_control_password()}\n\n self._log_line('Launching Tor (%s)' % self.tor_binary,\n notify=True)\n\n PresetSafePopenArgs(long_running=True)\n self.tor_process = stem.process.launch_tor_with_config(\n timeout=None, # Required or signal.signal will raise\n tor_cmd=self.tor_binary,\n config= tor_process_config,\n init_msg_handler=self._log_line)\n\n ctrl = stem.control.Controller.from_port(port=self.control_port)\n ctrl.authenticate(password=self.control_password)\n self.tor_controller = ctrl\n\n self.event.flags = Event.RUNNING\n self._log_line('Tor is live on socks=%d, control=%d'\n % (self.socks_port, self.control_port),\n notify=True)\n finally:\n MakePopenSafe()\n\n # Relaunch all the hidden services, if our Tor process died on us.\n self.relaunch_hidden_services()\n\n # Invoke any on-startup callbacks\n for cb in (self.callbacks or []):\n try:\n cb(self)\n except:\n self._log_line('Callback %s failed: %s'\n % (cb, traceback.format_exc()),\n notify=True)\n\n # Finally, just wait until our child process terminates.\n try:\n self.tor_process.wait()\n except:\n pass\n finally:\n self.event.flags = Event.COMPLETE\n self._log_line('Shut down', notify=True)\n self.tor_controller = None\n self.tor_process = None\n\n def _hashed_control_password(self):\n try:\n hasher = subprocess.Popen(\n [self.tor_binary, '--hush', '--hash-password',\n str(self.control_password)],\n stdout=subprocess.PIPE, bufsize=1)\n hasher.wait()\n expr = re.compile('([\\d]{2}:[\\w]{58})')\n match = filter(None, map(expr.match, hasher.stdout))[0]\n passhash = match.group(1)\n return passhash\n except:\n return None\n\n def _log_line(self, line, notify=False):\n if self.session:\n if notify:\n self.session.ui.notify(line)\n else:\n self.session.ui.debug(line)\n else:\n debug('%s' % line)\n\n log = self.event.data.get('log', [])\n log.append(line.strip())\n if len(log) > 100:\n log[:10] = []\n self.event.data['log'] = log\n if notify:\n self.event.message = line\n if self.config and self.config.event_log:\n self.config.event_log.log_event(self.event)\n\n def relaunch_hidden_services(self):\n hidden_services = copy.copy(self.hidden_services)\n for onion, (portmap, key_t, key_c) in hidden_services.iteritems():\n if key_t and key_c:\n self.launch_hidden_service(portmap, key_t, key_c)\n else:\n self._log_line('Failed to relaunch: %s' % onion, notify=True)\n\n def launch_hidden_service(self, portmap, key_type=None, key_content=None):\n with self.lock:\n aor = self.tor_controller.create_ephemeral_hidden_service(\n portmap,\n key_type=key_type if (key_content and key_type) else 'NEW',\n key_content=key_content or 'BEST',\n detached=True, await_publication=True)\n\n self.hidden_services[aor.service_id] = (\n portmap,\n aor.private_key_type or key_type,\n aor.private_key or key_content)\n self._log_line('Listening on Onion: %s.onion' % aor.service_id,\n notify=True)\n return aor\n\n def stop_tor(self, wait=True):\n self.keep_looping = False\n if self.tor_process is not None:\n try:\n for onion in self.hidden_services:\n try:\n t.tor_controller.remove_ephemeral_hidden_service(onion)\n except:\n pass\n\n with self.lock:\n self.tor_process.kill()\n if wait:\n self.tor_process.wait()\n except:\n pass\n Tor._instance = None\n\n def isReady(self, wait=False):\n while True:\n if ((self.tor_process is not None) and\n (self.tor_controller is not None)):\n return True\n if not wait:\n return False\n with self.lock:\n # If we have the lock, but self.tor_process is None, that\n # means startup has failed or we are dead - stop waiting!\n wait = (not self.started) or (self.tor_process is not None)\n if not self.started:\n time.sleep(0.1)\n\n def isAlive(self):\n # FIXME: This is inaccurate\n return (self.tor_process is not None)\n\n def quit(self, join=True):\n self.stop_tor(wait=join)\n\n\nif __name__ == \"__main__\":\n TEST_KEY = \"RSA1024:MIICXQIBAAKBgQDQ/+aYFpSvZZ5Ce2cpsuJz1epCcY9n+HZx/bC/D7mqEXCdDB9W13FMuwwK9FvjjXdfJzkdJ1GEcppEzd69C5xPZo2k+klKDhMONYhGHcm+CGu+JWNbqrcInNfZageu1Hg8g5Kz2h+/xCmuqKLSxGwJGvIoYfZupyn3DaxGnZv/2QIDAQABAoGAYs13L9MM+1Yo2PkJrhbZIzWvhzW0O8ykAgOSeOBwP0v7VuMSNbWn5ERQzyTyA8Mu+ZbLU1LxIJIlB/3jHK/Odoe2kkPjjaeKKVXGM+NMefps/YPs8abql06YoWN6KshY0BYkzkmlF/Xxl4t+jjvDG9Fsx6kJV6LKRwm6BFVzUTkCQQD2ujGQs1I1fsuCCHZXcnyoLO/hJaJLoj3clCiYcWhnfdpgkv0+CdIYE+DPZNsiIfruQQYZzZjKtO2xx0fvqKuPAkEA2Nq46nR1L0ISJizSfTYkz+KXuV8kgkMxwzkZC6l+DAqf4qxjFcDOwrIw9f75N8DcveHLD4R/fyMaesW6SK8KFwJAZ4Om1/bkPt17tIqoW/gEpOp1mhiYBvOC0NC4V3z9OK5suKfy59xm8QMmBt1hsuhexycw0BKaUDGoqDXb0IkLsQJBANQaUsWXdMrtX90Q+CxaGfVvVyGL6qSyXmjpXxLmDBBxD+Ng42VyeYk7SuJBKreanw3mXHvoB+BtkEfHQCY5dq8CQQCofoToQr5mTrlomus6/ei22Ein/BS9s0YUPCOpMkZfSp/GaWyEH7QjxatM/LoaMRlH/Y/wGMEK8P05F9DGBtSP\"\n t = Tor()\n try:\n debug_target = open( sys.argv[1], 'w' )\n except IndexError:\n pass\n try:\n debug( \"*** Starting Tor\" )\n t.start()\n if t.isReady(wait=True):\n debug( \"*** Creating hidden services...\" )\n key_type, key_content = TEST_KEY.split(':', 1)\n aor = t.launch_hidden_service({80: 80}, key_type, key_content)\n aor = t.launch_hidden_service(443)\n\n #raw_input(\"*** Hit enter to disable service and shutdown ***\")\n debug(\"waiting...\")\n time.sleep(5)\n finally:\n debug(\"quiting!?!\")\n t.quit()\n debug(\"exiting!?!\")\n" }, { "path": "mailpile/eventlog.py", "content": "from __future__ import print_function\nimport copy\nimport datetime\nimport json\nimport os\nimport threading\nimport time\nfrom email.utils import formatdate, parsedate_tz, mktime_tz\n\nfrom mailpile.crypto.streamer import EncryptingStreamer, DecryptingStreamer\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.util import EventRLock, EventLock, CleanText, json_helper\nfrom mailpile.util import safe_remove, thread_context\n\n\nEVENT_COUNTER_LOCK = threading.Lock()\nEVENT_COUNTER = 0\n\n\n\ndef NewEventId():\n \"\"\"\n This is guaranteed to generate unique event IDs for up to 1 million\n events per second. Beyond that, all bets are off. :-P\n \"\"\"\n global EVENT_COUNTER\n with EVENT_COUNTER_LOCK:\n EVENT_COUNTER = EVENT_COUNTER+1\n EVENT_COUNTER %= 0x100000\n return '%8.8x-%5.5x-%x' % (time.time(), EVENT_COUNTER, os.getpid())\n\n\ndef _ClassName(obj, ignore_regexps=False):\n if isinstance(obj, (str, unicode)):\n return str(obj).replace('mailpile.', '.')\n elif hasattr(obj, '__classname__'):\n return str(obj.__classname__).replace('mailpile.', '.')\n elif ignore_regexps and 'SRE_Pattern' in str(obj.__class__):\n return obj\n else:\n module = str(obj.__class__.__module__)\n if module.startswith('mailpile.'):\n module = module[len('mailpile'):]\n return '%s.%s' % (module, str(obj.__class__.__name__))\n\n\nclass Event(object):\n \"\"\"\n This is a single event in the event log. Actual interpretation and\n rendering of events should be handled by the respective source class.\n \"\"\"\n RUNNING = 'R'\n COMPLETE = 'c'\n INCOMPLETE = 'i'\n FUTURE = 'F'\n\n # For now these live here, we may templatize this later.\n PREAMBLE_HTML = '
    '\n PUBLIC_HTML = ('
  • %(date)s '\n '%(message)s
    • ')\n PRIVATE_HTML = PUBLIC_HTML\n POSTAMBLE_HTML = '
'\n\n @classmethod\n def Parse(cls, json_string):\n try:\n return cls(*json.loads(json_string))\n except:\n return cls()\n\n def __init__(self,\n ts=None, event_id=None, flags='c', message='',\n source=None, data=None, private_data=None):\n self._data = [\n '',\n (event_id or NewEventId()).replace('.', '-'),\n flags,\n message,\n _ClassName(source),\n data or {},\n private_data or {},\n ]\n self._set_ts(ts or time.time())\n\n def __str__(self):\n return json.dumps(self._data, default=json_helper)\n\n def _set_ts(self, ts):\n if hasattr(ts, 'timetuple'):\n self._ts = int(time.mktime(ts.timetuple()))\n elif isinstance(ts, (str, unicode)):\n self._ts = int(mktime_tz(parsedate_tz(ts)))\n else:\n self._ts = float(ts)\n self._data[0] = formatdate(self._ts)\n\n def _set(self, col, value):\n self._set_ts(time.time())\n self._data[col] = value\n\n def _get_source_class(self):\n try:\n module_name, class_name = CleanText(self.source,\n banned=CleanText.NONDNS\n ).clean.rsplit('.', 1)\n if module_name.startswith('.'):\n module_name = 'mailpile' + module_name\n module = __import__(module_name, globals(), locals(), class_name)\n return getattr(module, class_name)\n except (ValueError, AttributeError, ImportError):\n return None\n\n date = property(lambda s: s._data[0], lambda s, v: s._set_ts(v))\n ts = property(lambda s: s._ts, lambda s, v: s._set_ts(v))\n event_id = property(lambda s: s._data[1], lambda s, v: s._set(1, v))\n flags = property(lambda s: s._data[2], lambda s, v: s._set(2, v))\n message = property(lambda s: s._data[3], lambda s, v: s._set(3, v))\n source = property(lambda s: s._data[4],\n lambda s, v: s._set(4, _ClassName(v)))\n data = property(lambda s: s._data[5], lambda s, v: s._set(5, v))\n private_data = property(lambda s: s._data[6], lambda s, v: s._set(6, v))\n source_class = property(_get_source_class)\n\n def as_dict(self, private=True):\n try:\n return self.source_class.EventAsDict(self, private=private)\n except (AttributeError, NameError):\n data = {\n 'ts': self.ts,\n 'date': self.date,\n 'event_id': self.event_id,\n 'message': self.message,\n 'flags': self.flags,\n 'source': self.source,\n 'data': self.data\n }\n if private:\n data['private_data'] = self.private_data\n return data\n\n def as_text(self, private=True, compact=False):\n try:\n return self.source_class.EventAsText(self, private=private,\n compact=True)\n except (AttributeError, NameError):\n if compact:\n return '%s=%s:%s %s' % (self.event_id,\n self.source.split('.')[-1],\n self.flags, self.message)\n else:\n return json.dumps(self.as_dict(private=private),\n default=json_helper)\n\n def as_json(self, private=True):\n try:\n return self.source_class.EventAsJson(self, private=private)\n except (AttributeError, NameError):\n return json.dumps(self.as_dict(private=private))\n\n def as_html(self, private=True):\n try:\n return self.source_class.EventAsHtml(self, private=private)\n except (AttributeError, NameError):\n if private:\n return self.PRIVATE_HTML % self.as_dict(private=True)\n else:\n return self.PUBLIC_HTML % self.as_dict(private=False)\n\n\ndef GetThreadEvent(create=False, message=None, source=None):\n ctx = thread_context()\n if ctx and 'event' in ctx[-1]:\n return ctx[-1]['event']\n elif create:\n return Event(message=message, source=source)\n else:\n return None\n\n\nclass EventLog(object):\n \"\"\"\n This is the Mailpile Event Log.\n\n The log is written encrypted to disk on an ongoing basis (rotated\n every N lines), but entries are kept in RAM as well. The event log\n allows for recording of incomplete events, to help different parts\n of the app \"remember\" tasks which have yet to complete or may need\n to be retried.\n \"\"\"\n KEEP_LOGS = 2\n\n def __init__(self, logdir, decryption_key_func, encryption_key_func,\n rollover=1024):\n self.logdir = logdir\n self.decryption_key_func = decryption_key_func or (lambda: None)\n self.encryption_key_func = encryption_key_func or (lambda: None)\n self.rollover = rollover\n\n self._events = {}\n\n # Internals...\n self._watching_uis = []\n self._waiter = threading.Condition(EventRLock())\n self._lock = EventLock()\n self._log_fd = None\n\n def _notify_waiters(self):\n with self._waiter:\n self._waiter.notifyAll()\n\n def wait(self, timeout=None):\n with self._waiter:\n self._waiter.wait(timeout)\n\n def _save_filename(self):\n return os.path.join(self.logdir, self._log_start_id)\n\n def _open_log(self):\n if self._log_fd:\n self._log_fd.close()\n\n if not os.path.exists(self.logdir):\n os.mkdir(self.logdir)\n\n self._log_start_id = NewEventId()\n enc_key = self.encryption_key_func()\n if enc_key:\n self._log_fd = EncryptingStreamer(enc_key,\n dir=self.logdir,\n name='EventLog/ES',\n use_filter=False,\n long_running=True)\n self._log_fd.save(self._save_filename(), finish=False)\n self._log_write = self._log_fd.write_pad_and_flush\n else:\n self._log_fd = open(self._save_filename(), 'wb', 0)\n self._log_write = self._log_fd.write\n\n # Write any incomplete events to the new file\n for e in self.incomplete():\n self._log_write('%s\\n')\n\n # We're starting over, incomplete events don't count\n self._logged = 0\n\n def _maybe_rotate_log(self):\n if self._logged > self.rollover:\n self._log_fd.close()\n kept_events = {}\n for e in self.incomplete():\n kept_events[e.event_id] = e\n self._events = kept_events\n self._open_log()\n self.purge_old_logfiles()\n\n def _list_logfiles(self):\n return sorted([l for l in os.listdir(self.logdir)\n if not l.startswith('.')])\n\n def _save_events(self, events, recursed=False):\n if not self._log_fd:\n self._open_log()\n events.sort(key=lambda ev: ev.ts)\n try:\n for event in events:\n self._log_write('%s\\n' % event)\n self._events[event.event_id] = event\n except IOError:\n if recursed:\n raise\n else:\n self._unlocked_close()\n return self._save_events(events, recursed=True)\n\n def _load_logfile(self, lfn):\n enc_key = self.decryption_key_func()\n with open(os.path.join(self.logdir, lfn)) as fd:\n if enc_key:\n with DecryptingStreamer(fd, mep_key=enc_key,\n name='EventLog/DS(%s)' % lfn\n ) as streamer:\n lines = streamer.read()\n streamer.verify(_raise=IOError)\n else:\n lines = fd.read()\n if lines:\n for line in lines.splitlines():\n event = Event.Parse(line.strip())\n self._events[event.event_id] = event\n\n def _match(self, event, filters):\n def compare(val, rule):\n if isinstance(rule, (str, unicode)):\n return unicode(val) == unicode(rule)\n else:\n return rule.match(unicode(val)) is not None\n for kw, rule in filters.iteritems():\n if kw.endswith('!'):\n truth, okw, kw = False, kw, kw[:-1]\n else:\n truth, okw = True, kw\n if kw == 'source':\n if truth != compare(event.source,\n _ClassName(rule, ignore_regexps=True)):\n return False\n elif kw == 'flag':\n if truth != (rule in event.flags):\n return False\n elif kw == 'flags':\n if truth != compare(event.flags, rule):\n return False\n elif kw == 'event_id':\n if truth != compare(event.event_id, rule):\n return False\n elif kw == 'since':\n when = float(rule)\n if when < 0:\n when += time.time()\n if truth != (event.ts > when):\n return False\n elif kw.startswith('data_'):\n if truth != compare(event.data.get(kw[5:]), rule):\n return False\n elif kw.startswith('private_data_'):\n if truth != compare(event.data.get(kw[13:]), rule):\n return False\n else:\n # Unknown keywords match nothing...\n print('Unknown keyword: `%s=%s`' % (okw, rule))\n return False\n return True\n\n def incomplete(self, **filters):\n \"\"\"Return all the incomplete events, in order.\"\"\"\n if 'event_id' in filters:\n ids = [filters['event_id']]\n else:\n ids = sorted(self._events.keys())\n for ek in ids:\n e = self._events.get(ek, None)\n if (e is not None and\n Event.COMPLETE not in e.flags and\n self._match(e, filters)):\n yield e\n\n def since(self, ts, **filters):\n \"\"\"Return all events since a given time, in order.\"\"\"\n if ts < 0:\n ts += time.time()\n if 'event_id' in filters and filters['event_id'][:1] != '!':\n ids = [filters['event_id']]\n else:\n ids = sorted(self._events.keys())\n for ek in ids:\n e = self._events.get(ek, None)\n if (e is not None and\n e.ts >= ts and\n self._match(e, filters)):\n yield e\n\n def events(self, **filters):\n return self.since(0, **filters)\n\n def get(self, event_id, default=None):\n return self._events.get(event_id, default)\n\n def log_event(self, event):\n \"\"\"Log an Event object.\"\"\"\n with self._lock:\n self._save_events([event])\n self._logged += 1\n self._maybe_rotate_log()\n self._notify_waiters()\n for ui in self._watching_uis:\n ui.notify(event.as_text(compact=True))\n return event\n\n def log(self, *args, **kwargs):\n \"\"\"Log a new event.\"\"\"\n return self.log_event(Event(*args, **kwargs))\n\n def close(self):\n with self._lock:\n return self._unlocked_close()\n\n def _unlocked_close(self):\n try:\n self._log_fd.close()\n self._log_fd = None\n except (OSError, IOError):\n pass\n\n def _prune_completed(self):\n for event_id in self._events.keys():\n if Event.COMPLETE in self._events[event_id].flags:\n del self._events[event_id]\n\n def ui_watch(self, ui):\n while ui.log_parent is not None:\n ui = ui.log_parent\n if ui not in self._watching_uis:\n self._watching_uis.append(ui)\n return True\n else:\n return False\n\n def ui_unwatch(self, ui):\n while ui.log_parent is not None:\n ui = ui.log_parent\n try:\n self._watching_uis.remove(ui)\n except ValueError:\n pass\n\n def load(self):\n with self._lock:\n self._open_log()\n for lf in self._list_logfiles()[-4:]:\n try:\n self._load_logfile(lf)\n except (OSError, IOError):\n # Nothing we can do, no point complaining...\n pass\n self._prune_completed()\n self._save_events(self._events.values())\n return self\n\n def purge_old_logfiles(self, keep=None):\n keep = keep or self.KEEP_LOGS\n for lf in self._list_logfiles()[:-keep]:\n try:\n safe_remove(os.path.join(self.logdir, lf))\n except OSError:\n pass\n" }, { "path": "mailpile/httpd.py", "content": "#\n# Mailpile's built-in HTTPD\n#\n###############################################################################\nimport Cookie\nimport cStringIO\nimport hashlib\nimport gzip\nimport mimetypes\nimport os\nimport random\nimport select\nimport socket\nimport SocketServer\nimport time\nimport threading\nimport traceback\nfrom SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler\nfrom urllib import quote, unquote\nfrom urlparse import parse_qs, urlparse\n\nimport mailpile.util\nimport mailpile.security as security\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.urlmap import UrlMap\nfrom mailpile.util import *\nfrom mailpile.ui import *\n\nglobal WORD_REGEXP, STOPLIST, BORING_HEADERS, DEFAULT_PORT\n\nDEFAULT_PORT = 33411\n\nBLOCK_HTTPD_LOCK = UiRLock()\nLIVE_HTTP_REQUESTS = 0\n\n\ndef Idle_HTTPD(allowed=1):\n with BLOCK_HTTPD_LOCK:\n sleep = 100\n while (sleep and\n not mailpile.ui.QUITTING and\n LIVE_HTTP_REQUESTS > allowed):\n time.sleep(0.05)\n sleep -= 1\n return BLOCK_HTTPD_LOCK\n\n\nclass HttpRequestHandler(SimpleXMLRPCRequestHandler):\n # Allow persistent HTTP/1.1 connections\n protocol_version = 'HTTP/1.1'\n\n # We always recognize these extensions, no matter what the Python\n # mimetype module thinks.\n _MIMETYPE_MAP = dict([(ext, 'text/plain') for ext in (\n 'c', 'cfg', 'conf', 'cpp', 'csv', 'h', 'hpp', 'log', 'md', 'me',\n 'py', 'rb', 'rc', 'txt'\n )] + [(ext, 'application/x-font') for ext in (\n 'pfa', 'pfb', 'gsf', 'pcf'\n )] + [\n ('css', 'text/css'),\n ('eot', 'application/vnd.ms-fontobject'),\n ('gif', 'image/gif'),\n ('html', 'text/html'),\n ('htm', 'text/html'),\n ('ico', 'image/x-icon'),\n ('jpg', 'image/jpeg'),\n ('jpeg', 'image/jpeg'),\n ('js', 'text/javascript'),\n ('json', 'application/json'),\n ('otf', 'font/otf'),\n ('png', 'image/png'),\n ('rss', 'application/rss+xml'),\n ('tif', 'image/tiff'),\n ('tiff', 'image/tiff'),\n ('ttf', 'font/ttf'),\n ('svg', 'image/svg+xml'),\n ('svgz', 'image/svg+xml'),\n ('woff', 'application/font-woff'),\n ])\n\n _ERROR_CONTEXT = {'lastq': '', 'csrf': '', 'path': ''},\n _NEWLINE_RE = re.compile('[\\r\\n]+')\n _HTML_RE = re.compile('[<>\\'\\\"]+')\n\n def assert_no_newline(self, data):\n if re.search(self._NEWLINE_RE, str(data) or '') is not None:\n raise ValueError()\n\n def assert_no_html(self, data):\n if re.search(self._HTML_RE, data or '') is not None:\n raise ValueError()\n\n def send_header(self, hdr, value):\n self.assert_no_newline(value)\n return SimpleXMLRPCRequestHandler.send_header(self, hdr, value)\n\n def http_host(self):\n \"\"\"Return the current server host, e.g. 'localhost'\"\"\"\n try:\n # rsplit removes port\n return self.headers.get('host', 'localhost').rsplit(':', 1)[0]\n except AttributeError:\n return 'unknown'\n\n def _load_cookies(self):\n \"\"\"Robustified cookie parser that silently drops invalid cookies.\"\"\"\n cookies = Cookie.SimpleCookie()\n for fragment in self.headers.get('cookie', '').split('; '):\n if fragment:\n try:\n cookies.load(fragment)\n except Cookie.CookieError:\n pass\n return cookies\n\n def http_session(self):\n \"\"\"Fetch the session ID from a cookie, or assign a new one\"\"\"\n session_id = self._load_cookies().get(self.server.session_cookie)\n if session_id:\n session_id = session_id.value\n self.assert_no_newline(session_id)\n else:\n session_id = self.server.make_session_id(self)\n return session_id\n\n def server_url(self):\n \"\"\"Return the current server URL, e.g. 'http://localhost:33411/'\"\"\"\n try:\n surl = '%s://%s' % (self.headers.get('x-forwarded-proto', 'http'),\n self.headers.get('host', 'localhost'))\n self.server.server_url = surl\n except AttributeError:\n surl = self.server.server_url\n return surl\n\n def send_http_response(self, code, msg):\n \"\"\"Send the HTTP response header\"\"\"\n msg = '%s %s' % (code, msg)\n self.assert_no_newline(msg)\n self.wfile.write('HTTP/1.1 %s\\r\\n' % msg)\n\n def send_http_redirect(self, destination):\n # We don't re-encode things here, we expect our input to already\n # be well formed. However, this is the last chance to block any\n # exploits, so we do check to make sure.\n self.assert_no_newline(destination)\n self.assert_no_html(destination)\n self.send_http_response(302, 'Found')\n body = ('

Please look here!

\\n'\n ) % (destination,)\n self.wfile.write(('Location: %s\\r\\n'\n 'Content-Length: %d\\r\\n\\r\\n'\n ) % (destination, len(body)))\n self.wfile.write(body)\n\n def send_standard_headers(self,\n header_list=[],\n cachectrl='private',\n mimetype='text/html',\n x_dns_prefetch='off'):\n \"\"\"\n Send common HTTP headers plus a list of custom headers:\n - Cache-Control\n - Content-Type\n - X-DNS-Prefetch-Control\n\n This function does not send the HTTP/1.1 header, so\n ensure self.send_http_response() was called before\n\n Keyword arguments:\n header_list -- A list of custom headers to send, containing\n key-value tuples\n cachectrl -- The value of the 'Cache-Control' header field\n mimetype -- The MIME type to send as 'Content-Type' value\n \"\"\"\n if mimetype.startswith('text/') and ';' not in mimetype:\n mimetype += ('; charset = utf-8')\n self.send_header('Cache-Control', cachectrl)\n self.send_header('Content-Security-Policy',\n security.http_content_security_policy(self.server))\n self.send_header('Content-Type', mimetype)\n self.send_header('X-DNS-Prefetch-Control', x_dns_prefetch)\n self.send_header('X-UA-Compatible', 'IE=Edge') # For old Windowses\n for header in header_list:\n self.send_header(header[0], header[1])\n session_id = self.session.ui.html_variables.get('http_session')\n if session_id:\n cookies = Cookie.SimpleCookie()\n cookies[self.server.session_cookie] = session_id\n cookies[self.server.session_cookie]['path'] = '/'\n cookies[self.server.session_cookie]['max-age'] = 24 * 3600\n self.send_header(*cookies.output().split(': ', 1))\n if mailpile.util.QUITTING:\n self.send_header('Connection', 'close')\n self.end_headers()\n\n def send_full_response(self, message,\n code=200, msg='OK',\n mimetype='text/html', header_list=[],\n cachectrl=None,\n suppress_body=False):\n \"\"\"\n Sends the HTTP header and a response list\n\n message -- The body of the response to send\n header_list -- A list of custom headers to send,\n containing key-value tuples\n code -- The HTTP response code to send\n mimetype -- The MIME type to send as 'Content-Type' value\n suppress_body -- Set this to True to ignore the message parameter\n and not send any response body\n \"\"\"\n message = unicode(message).encode('utf-8')\n self.log_request(code, message and len(message) or '-')\n # Send HTTP/1.1 header\n self.send_http_response(code, msg)\n # Send all headers\n if code == 401:\n self.send_header('WWW-Authenticate',\n 'Basic realm = MP%d' % (time.time() / 3600))\n # If suppress_body == True, we don't know the content length\n headers = []\n if not suppress_body:\n message, headers = self._maybe_gzip(message, len(message or ''), [])\n self.send_standard_headers(header_list=(header_list + headers),\n mimetype=mimetype,\n cachectrl=(cachectrl or \"no-cache\"))\n # Response body\n if not suppress_body:\n self.wfile.write(message or '')\n\n def guess_mimetype(self, fpath):\n ext = os.path.basename(fpath).rsplit('.')[-1]\n return (self._MIMETYPE_MAP.get(ext.lower()) or\n mimetypes.guess_type(fpath, strict=False)[0] or\n 'application/octet-stream')\n\n def _mk_etag(self, *args):\n # This ETag varies by whatever args we give it (e.g. size, mtime,\n # etc), but is unique per Mailpile instance and should leak nothing\n # about the actual server configuration.\n data = '%s-%s' % (self.server.secret, '-'.join((str(a) for a in args)))\n return hashlib.md5(data).hexdigest()\n\n def _maybe_gzip(self, data, msg_size, headers):\n if (data and\n (len(data) > 1400) and\n (data[:2] not in ('\\xff\\xd8', '\\x89\\x50', # JPEG, PNG\n '\\x1f\\x8b', 'BZ', 'PK' # GZIP, BZIP, PKZIP\n )) and\n ('gzip' in self.headers.get('accept-encoding', ''))):\n gzipped = cStringIO.StringIO()\n with gzip.GzipFile(fileobj=gzipped, mode='w') as fd:\n fd.write(data)\n gzipped = gzipped.getvalue()\n if len(data) > len(gzipped):\n headers.extend([('Content-Length', '%s' % len(gzipped)),\n ('X-Full-Size', '%s' % msg_size),\n ('Content-Encoding', 'gzip')])\n return gzipped, headers\n headers.append(('Content-Length', '%s' % msg_size))\n return data, headers\n\n def send_file(self, config, filename, suppress_body=False):\n # FIXME: Do we need more security checks?\n if '..' in filename:\n code, msg = 403, \"Access denied\"\n else:\n try:\n tpl = config.sys.path.get(self.http_host(), 'html_theme')\n fpath, fd, mt = config.open_file(tpl, filename)\n with fd:\n mimetype = mt or self.guess_mimetype(fpath)\n msg_size = os.path.getsize(fpath)\n if not suppress_body:\n message = fd.read()\n else:\n message = None\n code, msg = 200, \"OK\"\n except IOError as e:\n mimetype = 'text/plain'\n if e.errno == 2:\n code, msg = 404, \"File not found\"\n elif e.errno == 13:\n code, msg = 403, \"Access denied\"\n else:\n code, msg = 500, \"Internal server error\"\n message = None\n msg_size = 0\n\n # Note: We assume the actual static content almost never varies\n # on a given Mailpile instance, thuse the long TTL and no\n # ETag for conditional loads.\n\n message, headers = self._maybe_gzip(message, msg_size, [])\n self.log_request(code, msg_size if (message is not None) else '-')\n self.send_http_response(code, msg)\n self.send_standard_headers(header_list=headers,\n mimetype=mimetype,\n cachectrl='must-revalidate, max-age=36000')\n self.wfile.write(message or '')\n\n def do_POST(self, method='POST'):\n (scheme, netloc, path, params, query, frag) = urlparse(self.path)\n if path.startswith('/::XMLRPC::/'):\n raise ValueError(_('XMLRPC has been disabled for now.'))\n #return SimpleXMLRPCRequestHandler.do_POST(self)\n\n # Update thread name for debugging purposes\n threading.current_thread().name = 'POST:%s' % self.path.split('?')[0]\n\n self.session, config = self.server.session, self.server.session.config\n post_data = {}\n try:\n ue = 'application/x-www-form-urlencoded'\n clength = int(self.headers.get('content-length', 0))\n ctype, pdict = cgi.parse_header(self.headers.get('content-type',\n ue))\n if ctype == 'multipart/form-data':\n post_data = cgi.FieldStorage(\n fp=self.rfile,\n headers=self.headers,\n environ={'REQUEST_METHOD': method,\n 'CONTENT_TYPE': self.headers['Content-Type']}\n )\n elif ctype == ue:\n if clength > 5 * 1024 * 1024:\n raise ValueError(_('OMG, input too big'))\n post_data = cgi.parse_qs(self.rfile.read(clength), 1)\n else:\n raise ValueError(_('Unknown content-type'))\n\n except (IOError, ValueError) as e:\n self.send_full_response(self.server.session.ui.render_page(\n config, self._ERROR_CONTEXT,\n body='POST geborked: %s' % e,\n title=_('Internal Error')\n ), code=500)\n return None\n return self.do_GET(post_data=post_data, method=method)\n\n def do_GET(self, *args, **kwargs):\n global LIVE_HTTP_REQUESTS\n try:\n path = self.path.split('?')[0]\n\n threading.current_thread().name = 'WAIT:%s' % path\n with BLOCK_HTTPD_LOCK:\n LIVE_HTTP_REQUESTS += 1\n\n threading.current_thread().name = 'WORK:%s' % path\n return self._real_do_GET(*args, **kwargs)\n finally:\n threading.current_thread().name = 'DONE:%s' % path\n LIVE_HTTP_REQUESTS -= 1\n if mailpile.util.QUITTING:\n self.wfile.close()\n\n def _real_do_GET(self, post_data={}, suppress_body=False, method='GET'):\n (scheme, netloc, path, params, query, frag) = urlparse(self.path)\n query_data = parse_qs(query)\n opath = path = unquote(path)\n\n # HTTP is stateless, so we create a new session for each request.\n self.session, config = self.server.session, self.server.session.config\n server_session = self.server.session\n\n # Debugging...\n if 'httpdata' in config.sys.debug:\n self.wfile = DebugFileWrapper(sys.stderr, self.wfile)\n\n # Path manipulation...\n if path == '/favicon.ico':\n path = '%s/static/favicon.ico' % (config.sys.http_path or '')\n if config.sys.http_path:\n if not path.startswith(config.sys.http_path):\n self.send_full_response(_(\"File not found (invalid path)\"),\n code=404, mimetype='text/plain')\n return None\n path = path[len(config.sys.http_path):]\n if path.startswith('/_/'):\n path = path[2:]\n for static in ('/static/', '/bower_components/'):\n if path.startswith(static):\n return self.send_file(config, path[len(static):],\n suppress_body=suppress_body)\n\n self.session = session = Session(config)\n session.ui = HttpUserInteraction(self, config,\n log_parent=server_session.ui)\n if 'context' in post_data:\n session.load_context(post_data['context'][0])\n elif 'context' in query_data:\n session.load_context(query_data['context'][0])\n\n mark_name = 'Processing HTTP API request at %s' % time.time()\n session.ui.start_command(mark_name, [], {})\n\n if 'http' in config.sys.debug:\n session.ui.warning = server_session.ui.warning\n session.ui.notify = server_session.ui.notify\n session.ui.error = server_session.ui.error\n session.ui.debug = server_session.ui.debug\n session.ui.debug('%s: %s qs = %s post = %s'\n % (method, opath, query_data, post_data))\n\n idx = session.config.index\n if session.config.loaded_config:\n name = session.config.get_profile().get('name', 'Chelsea Manning')\n else:\n name = 'Chelsea Manning'\n\n http_headers = []\n http_session = self.http_session()\n csrf_token = security.make_csrf_token(self.server.secret, http_session)\n session.ui.html_variables = {\n 'csrf_token': csrf_token,\n 'csrf_field': (''\n % csrf_token),\n 'http_host': self.headers.get('host', 'localhost'),\n 'http_hostname': self.http_host(),\n 'http_method': method,\n 'http_session': http_session,\n 'http_request': self,\n 'http_response_headers': http_headers,\n 'message_count': (idx and len(idx.INDEX) or 0),\n 'name': name,\n 'title': 'Mailpile dummy title',\n 'url_protocol': self.headers.get('x-forwarded-proto', 'http'),\n 'mailpile_size': idx and len(idx.INDEX) or 0\n }\n session.ui.valid_csrf_token = lambda token: security.valid_csrf_token(\n self.server.secret, http_session, token)\n\n try:\n try:\n need_auth = not (mailpile.util.TESTING or\n session.config.sys.http_no_auth)\n commands = UrlMap(session).map(\n self, method, path, query_data, post_data,\n authenticate=need_auth)\n except UsageError:\n if (not path.endswith('/') and\n not session.config.sys.debug and\n method == 'GET'):\n commands = UrlMap(session).map(self, method, path + '/',\n query_data, post_data)\n url = quote(path) + '/'\n if query:\n url += '?' + query\n return self.send_http_redirect(url)\n else:\n raise\n\n cachectrl = None\n if 'http' not in config.sys.debug:\n etag_data = []\n max_ages = []\n have_ed = 0\n for c in commands:\n max_ages.append(c.max_age())\n ed = c.etag_data()\n have_ed += 1 if ed else 0\n etag_data.extend(ed)\n if have_ed == len(commands):\n etag = self._mk_etag(*etag_data)\n conditional = self.headers.get('if-none-match')\n if conditional == etag:\n self.send_full_response('OK', code=304,\n msg='Unmodified')\n return None\n else:\n http_headers.append(('ETag', etag))\n max_age = min(max_ages) if max_ages else 10\n if max_age:\n cachectrl = 'must-revalidate, max-age=%d' % max_age\n else:\n cachectrl = 'must-revalidate, no-store, max-age=0'\n\n global LIVE_HTTP_REQUESTS\n hang_fix = 1 if ([1 for c in commands if c.IS_HANGING_ACTIVITY]\n ) else 0\n try:\n LIVE_HTTP_REQUESTS -= hang_fix\n\n session.ui.mark('Running %d commands' % len(commands))\n results = [cmd.run() for cmd in commands]\n\n session.ui.mark('Displaying final result')\n session.ui.display_result(results[-1])\n finally:\n LIVE_HTTP_REQUESTS += hang_fix\n\n session.ui.mark('Rendering response')\n mimetype, content = session.ui.render_response(session.config)\n\n session.ui.mark('Sending response')\n self.send_full_response(content,\n mimetype=mimetype,\n header_list=http_headers,\n cachectrl=cachectrl)\n\n except UrlRedirectException as e:\n return self.send_http_redirect(e.url)\n except SuppressHtmlOutput:\n return None\n except AccessError:\n self.send_full_response(_('Access Denied'),\n code=403, mimetype='text/plain')\n return None\n except:\n e = traceback.format_exc()\n session.ui.debug(e)\n if not session.config.sys.debug:\n e = _('Internal Error')\n self.send_full_response(e, code=500, mimetype='text/plain')\n return None\n\n finally:\n session.ui.report_marks(\n details=('timing' in session.config.sys.debug))\n session.ui.finish_command(mark_name)\n\n def do_PUT(self):\n return self.do_POST(method='PUT')\n\n def do_UPDATE(self):\n return self.do_POST(method='UPDATE')\n\n def do_HEAD(self):\n return self.do_GET(suppress_body=True, method='HEAD')\n\n def log_message(self, fmt, *args):\n if 'http' in self.server.session.config.sys.debug:\n self.server.session.ui.notify(self.server_url() +\n ' ' + (fmt % args))\n\n\nclass HttpServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):\n def __init__(self, session, sspec, handler):\n SimpleXMLRPCServer.__init__(self, sspec[:2], handler)\n self.daemon_threads = True\n self.session = session\n self.sessions = {}\n self.session_cookie = None\n\n # Duplicates from SocketServer.py, so our overrides work\n self.__is_shut_down = threading.Event()\n self.__shutdown_request = False\n\n # This lets us create new HTTPDs withut waiting for this one to\n # completely shut down.\n self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n\n # We set a large sending buffer to avoid blocking, because the GIL and\n # scheduling interact badly when we have busy background jobs.\n self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 128 * 1024)\n self.server_url = 'http://UNKNOWN/'\n self.sspec = (sspec[0] or 'localhost',\n self.socket.getsockname()[1],\n sspec[2])\n\n # This hash includes the index ofuscation master key, which means\n # it should be very strongly unguessable.\n self.secret = okay_random(64, session.config.get_master_key())\n\n # Generate a new unguessable session cookie name on startup\n while not self.session_cookie:\n self.session_cookie = okay_random(12, self.secret)\n\n def serve_forever(self, poll_interval=0.5, tick_func=None):\n \"\"\"\n Override SocketServer.serve_forever to allow other things to happen.\n \"\"\"\n if self.__is_shut_down is None:\n return\n self.__is_shut_down.clear()\n try:\n while not (self.__shutdown_request or mailpile.util.QUITTING):\n # FIXME: Let's add a global FD to interrupt this, so we can\n # be more responsive AND lengthen our timeouts.\n r, w, e = SocketServer._eintr_retry(\n select.select, [self], [], [], poll_interval)\n if self in r:\n self._handle_request_noblock()\n elif not (mailpile.util.QUITTING or tick_func is None):\n tick_func(self)\n finally:\n self.__shutdown_request = False\n if self.__is_shut_down is not None:\n self.__is_shut_down.set()\n\n def shutdown(self, join=True):\n self.__shutdown_request = True\n if join and (self.__is_shut_down is not None):\n self.__is_shut_down.wait()\n self.__is_shut_down = None\n\n def make_session_id(self, request):\n \"\"\"Generate an unguessable and unauthenticated new session ID.\"\"\"\n session_id = None\n while session_id in self.sessions or session_id is None:\n session_id = okay_random(32, self.secret,\n '%s' % (request and request.headers))\n return session_id\n\n def finish_request(self, request, client_address):\n try:\n SimpleXMLRPCServer.finish_request(self, request, client_address)\n except (socket.error, AttributeError):\n # AttributeError may get thrown if the underlying socket has\n # already been closed elsewhere and _sock = None.\n pass\n finally:\n if mailpile.util.QUITTING:\n self.shutdown()\n\n\nclass HttpWorker(threading.Thread):\n def __init__(self, session, sspec):\n threading.Thread.__init__(self)\n self.httpd = HttpServer(session, sspec, HttpRequestHandler)\n self.daemon = True\n self.session = session\n\n\n def idle_tick(self, httpd):\n pass\n\n def run(self):\n while self.httpd is not None:\n try:\n self.httpd.serve_forever(\n poll_interval=1.0, tick_func=self.idle_tick)\n except KeyboardInterrupt:\n return\n except socket.error:\n pass\n except:\n time.sleep(1)\n if self.httpd:\n traceback.print_exc()\n\n def quit(self, join=False):\n if self.httpd:\n try:\n self.httpd.server_close()\n except (OSError, IOError):\n pass\n self.httpd.shutdown(join=join)\n self.httpd = None\n" }, { "path": "mailpile/i18n.py", "content": "import gettext\nimport os\nimport threading\nfrom gettext import translation, gettext, NullTranslations\nfrom jinja2 import Environment, BaseLoader, TemplateNotFound\n\n\nACTIVE_TRANSLATION = None\n\nRECENTLY_TRANSLATED_LOCK = threading.Lock()\nRECENTLY_TRANSLATED = []\n\nFORMAT_CHECKED = {}\n\n\n# This little doodad will on-the-fly check whether our translators\n# messed up our format strings in various ways, and suppress the\n# translation if it is obviously broken.\ndef _fmt_safe(translation, original):\n global FORMAT_CHECKED\n if translation in FORMAT_CHECKED:\n return FORMAT_CHECKED[translation]\n if '%' in original:\n try:\n safe_assert(len([c for c in translation if c == '%'])\n == len([c for c in original if c == '%']))\n bogon = translation % 1\n FORMAT_CHECKED[translation] = translation\n except TypeError:\n # This just means we gave the wrong argument or the wrong\n # number of arguments - so the format string itself is OK.\n FORMAT_CHECKED[translation] = translation\n except:\n FORMAT_CHECKED[translation] = original\n else:\n FORMAT_CHECKED[translation] = translation\n return FORMAT_CHECKED[translation]\n\n\ndef gettext(string):\n with RECENTLY_TRANSLATED_LOCK:\n if isinstance(string, str):\n global RECENTLY_TRANSLATED\n RECENTLY_TRANSLATED = [t for t in RECENTLY_TRANSLATED[-100:]\n if t != string] + [string]\n if not ACTIVE_TRANSLATION:\n return string\n\n # FIXME: What if our input is utf-8? Does gettext want us to\n # encode it first, or send the UTF-8 string? Since we are\n # not encoding it, the decode below may fail. :(\n translation = ACTIVE_TRANSLATION.org_gettext(string)\n try:\n translation = translation.decode('utf-8')\n except UnicodeEncodeError:\n pass\n\n return _fmt_safe(translation, string)\n\n\ndef ngettext(string1, string2, n):\n with RECENTLY_TRANSLATED_LOCK:\n global RECENTLY_TRANSLATED\n RECENTLY_TRANSLATED = [t for t in RECENTLY_TRANSLATED[-100:]\n if t not in (string1, string2)\n ] + [string1, string2]\n\n default = string1 if (n == 1) else string2\n if not ACTIVE_TRANSLATION:\n return default\n\n # FIXME: What if our input is utf-8? Does gettext want us to\n # encode it first, or send the UTF-8 string? Since we are\n # not encoding it, the decode below may fail. :(\n translation = ACTIVE_TRANSLATION.org_ngettext(string1, string2, n)\n try:\n translation = translation.decode('utf-8')\n except UnicodeEncodeError:\n pass\n\n return _fmt_safe(translation, default)\n\n\nclass i18n_disabler:\n def __init__(self):\n self.stack = []\n\n def __enter__(self):\n global ACTIVE_TRANSLATION\n self.stack.append(ACTIVE_TRANSLATION)\n ACTIVE_TRANSLATION = None\n\n def __exit__(self, *args, **kwargs):\n global ACTIVE_TRANSLATION\n ACTIVE_TRANSLATION = self.stack.pop(-1)\n\n\ni18n_disabled = i18n_disabler()\n\n\ndef ActivateTranslation(session, config, language, localedir=None):\n global ACTIVE_TRANSLATION, RECENTLY_TRANSLATED\n\n if not language:\n language = os.getenv('LANG', None)\n\n if not localedir:\n import mailpile.config.paths\n localedir = mailpile.config.paths.DEFAULT_LOCALE_DIRECTORY()\n\n trans = None\n if (not language) or language[:5].lower() in ('en', 'en_us', 'c'):\n trans = NullTranslations()\n elif language:\n try:\n trans = translation(\"mailpile\", localedir,\n [language], codeset=\"utf-8\")\n except IOError:\n if session:\n session.ui.debug('Failed to load language %s' % language)\n\n if not trans:\n trans = translation(\"mailpile\", localedir,\n codeset='utf-8', fallback=True)\n\n if session:\n session.ui.debug('Failed to configure i18n (%s). '\n 'Using fallback.' % language)\n\n if trans:\n with RECENTLY_TRANSLATED_LOCK:\n RECENTLY_TRANSLATED = []\n\n ACTIVE_TRANSLATION = trans\n trans.org_gettext = trans.gettext\n trans.org_ngettext = trans.ngettext\n trans.gettext = lambda t, g: gettext(g)\n trans.ngettext = lambda t, s1, s2, n: ngettext(s1, s2, n)\n trans.set_output_charset(\"utf-8\")\n\n if hasattr(config, 'jinja_env'):\n config.jinja_env.install_gettext_translations(trans,\n newstyle=True)\n\n if session and language and not isinstance(trans, NullTranslations):\n session.ui.debug(gettext('Loaded language %s') % language)\n\n return trans\n\n\ndef ListTranslations(config, localedir=None):\n if not localedir:\n import mailpile.config.paths\n localedir = mailpile.config.paths.DEFAULT_LOCALE_DIRECTORY()\n languages = {\n 'C': 'English (Mailpile default)'\n }\n for lang in os.listdir(localedir):\n langdir = os.path.join(localedir, lang, 'LC_MESSAGES')\n if not os.path.exists(os.path.join(langdir, 'mailpile.mo')):\n continue\n try:\n with open(os.path.join(langdir, 'mailpile.po')) as fd:\n for line in fd.read(8192).splitlines():\n line = line.decode('utf-8')\n if line[1:].startswith('Language-Team: '):\n languages[lang] = ' '.join([word for word in\n line[1:-2].split()[1:-1]]\n ).replace('LANGUAGE', lang)\n except (IOError, OSError, UnicodeDecodeError):\n pass\n return languages\n" }, { "path": "mailpile/index/__init__.py", "content": "" }, { "path": "mailpile/index/base.py", "content": "from __future__ import print_function\nimport copy\nimport json\nimport random\nimport rfc822\nimport time\nimport traceback\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.index.msginfo import MessageInfoConstants\nfrom mailpile.index.search import SearchResultSet\nfrom mailpile.mailutils import MBX_ID_LEN\nfrom mailpile.mailutils.addresses import AddressHeaderParser\nfrom mailpile.mailutils.safe import *\nfrom mailpile.util import *\n\n\nclass BaseIndex(MessageInfoConstants):\n\n MAX_CACHE_ENTRIES = 250\n\n CAN_SEARCH = 'can_search' # Can search message contents\n CAN_SORT = 'can_sort' # Can sort search results\n HAS_UNREAD = 'has_unread' # Can filter messages by read/unread\n HAS_ATTS = 'has_atts' # Can filter messages by attachments or no\n HAS_TAGS = 'has_tags' # Can filter messages by tags and/or apply tags\n\n # What is this Index capable of? A list or set of the above.\n CAPABILITIES = []\n\n def __init__(self, config):\n self.config = config\n self.CACHE = {}\n self.EMAILS = []\n self.EMAIL_IDS = {}\n\n ### Known e-mail addresses #############################################\n\n # NOTE: This is all probably a misfeature and should probably go away.\n\n def add_email(self, email, name=None, eid=None):\n if eid is None:\n eid = len(self.EMAILS)\n self.EMAILS.append('')\n self.EMAILS[eid] = '%s (%s)' % (email, name or email)\n self.EMAIL_IDS[email.lower()] = eid\n # FIXME: This needs to get written out...\n return eid\n\n def update_email(self, email, name=None, change_name=True):\n eid = self.EMAIL_IDS.get(email.lower())\n if (eid is not None) and not change_name:\n el = self.EMAILS[eid].split(' ')\n if len(el) == 2:\n en = el[1][1:-1]\n if '@' not in en:\n name = en\n return self.add_email(email, name=name, eid=eid)\n\n def compact_to_list(self, msg_to):\n eids = []\n for ai in msg_to:\n email = ai.address\n eid = self.EMAIL_IDS.get(email.lower())\n if eid is None:\n eid = self.add_email(email, name=ai.fn)\n elif ai.fn and ai.fn != email:\n self.update_email(email, name=ai.fn, change_name=False)\n eids.append(eid)\n return ','.join([b36(e) for e in set(eids)])\n\n def expand_to_list(self, msg_info, field=None):\n eids = msg_info[field if (field is not None) else self.MSG_TO]\n eids = [e for e in eids.strip().split(',') if e]\n return [self.EMAILS[int(e, 36)] for e in eids]\n\n\n ### Tags & filters #####################################################\n\n def remove_tag(self, session, tid, msg_idxs=None):\n pass\n\n def add_tag(self, session, tid, msg_idxs=None):\n pass\n\n def apply_filters(self, session, fid, msg_idxs=None):\n pass\n\n\n ### Searching & sorting ################################################\n\n def search(self, session, terms, context=None):\n return SearchResultSet(self, terms, [], [])\n\n def sort_results(self, session, results, sort_order):\n pass\n\n def get_conversation(self, msg_idx=None):\n return []\n\n\n ### Loading data: subclasses override these ############################\n\n def get_msg_at_idx_pos_uncached(self, msg_idx):\n raise IndexError('Unimplemented')\n\n def open_mailbox_by_ptr(self, msg_ptr):\n return self.config.open_mailbox(None, msg_ptr[:MBX_ID_LEN])\n\n\n ### Loading data: higher level methods #################################\n\n def unique_mbox_ids(self, msg_info):\n return set([\n p[:MBX_ID_LEN] for p in msg_info[self.MSG_PTRS].split(',') if p])\n\n def enumerate_ptrs_mboxes_fds(self, msg_info):\n for msg_ptr in self._sorted_msg_ptrs(msg_info):\n mbox = fd = None\n try:\n mbox = self.open_mailbox_by_ptr(msg_ptr)\n fd = mbox.get_file_by_ptr(msg_ptr)\n except (IOError, OSError, KeyError, ValueError, IndexError):\n if 'sources' in self.config.sys.debug:\n traceback.print_exc()\n print('WARNING: %s not found' % msg_ptr)\n yield (msg_ptr, mbox, fd)\n\n ### ... ################################################################\n\n def _sorted_msg_ptrs(self, msg_info):\n ptrs = (p.strip() for p in msg_info[self.MSG_PTRS].split(','))\n # FIXME: Prefer local data? Prefer some mailbox types? Hmm.\n # Doing this well would speed things up and ensure the\n # `message/delete --keep` deduplication works nicely.\n return sorted([p for p in ptrs if p])\n\n def _encode_msg_id(self, msg_id):\n \"\"\"Normalize and hash a message ID for the metadata index\"\"\"\n if '<' in msg_id:\n new_msg_id = '<%s>' % msg_id.split('<')[1].split('>')[0]\n if len(new_msg_id) > 2:\n msg_id = new_msg_id\n return b64c(sha1b64(msg_id.strip()))\n\n def get_msg_id(self, msg, msg_ptr):\n return self._encode_msg_id(safe_get_msg_id(msg) or msg_ptr)\n\n def _message_to_msg_info(self, msg_idx_pos, msg_ptr, msg):\n msg_mid = b36(msg_idx_pos)\n msg_to = AddressHeaderParser(msg.get('to'))\n msg_cc = AddressHeaderParser(msg.get('cc'))\n msg_cc += AddressHeaderParser(msg.get('bcc'))\n return [\n msg_mid,\n msg_ptr, # Message PTR\n self.get_msg_id(msg, msg_ptr), # Message ID\n b36(safe_message_ts(msg)), # Message timestamp\n safe_decode_hdr(msg, 'from'), # Message from\n self.compact_to_list(msg_to), # Compacted to-list\n self.compact_to_list(msg_cc), # Compacted cc/bcc-list\n b36(len(msg) // 1024), # Message size\n safe_decode_hdr(msg, 'subject'), # Subject\n self.MSG_BODY_LAZY, # Body snippets come later\n '', # Tags\n '', # Replies\n msg_mid] # Thread\n\n def get_msg_at_idx_pos(self, msg_idx):\n try:\n crv = self.CACHE.get(msg_idx, {})\n if 'msg_info' in crv:\n return crv['msg_info']\n\n if len(self.CACHE) > self.MAX_CACHE_ENTRIES:\n try:\n for k in random.sample(\n self.CACHE.keys(), self.MAX_CACHE_ENTRIES/20):\n del self.CACHE[k]\n except KeyError:\n pass\n rv = self.get_msg_at_idx_pos_uncached(msg_idx)\n crv['msg_info'] = rv\n self.CACHE[msg_idx] = crv\n return rv\n\n except (IndexError, ValueError):\n return copy.copy(self.BOGUS_METADATA)\n\n def set_msg_at_idx_pos(self, msg_idx, msg_info):\n pass\n\n return [] # FIXME\n\n @classmethod\n def get_body(self, msg_info):\n msg_body = msg_info[self.MSG_BODY]\n if msg_body.startswith('{'):\n if msg_body == self.MSG_BODY_LAZY:\n return {'snippet': _('(unprocessed)'), 'lazy': True}\n elif msg_body == self.MSG_BODY_GHOST:\n return {'snippet': _('(ghost)'), 'ghost': True}\n elif msg_body == self.MSG_BODY_DELETED:\n return {'snippet': _('(deleted)'), 'deleted': True}\n try:\n return json.loads(msg_body)\n except ValueError:\n pass\n return {\n 'snippet': msg_body\n }\n\n @classmethod\n def truncate_body_snippet(self, body, max_chars):\n if 'snippet' in body:\n delta = len(self.encode_body(body)) - max_chars\n if delta > 0:\n body['snippet'] = body['snippet'][:-delta].rsplit(' ', 1)[0]\n\n @classmethod\n def encode_body(self, d, **kwargs):\n for k, v in kwargs:\n if v is None:\n if k in d:\n del d[k]\n else:\n d[k] = v\n if len(d) == 1 and 'snippet' in d:\n snippet = d['snippet']\n if snippet[:3] in self.MSG_BODY_MAGIC or snippet[:1] != '{':\n return d['snippet']\n return json.dumps(d, indent=None, separators=(',', ':'))\n\n @classmethod\n def set_body(self, msg_info, **kwargs):\n d = self.get_body(msg_info)\n msg_info[self.MSG_BODY] = self.encode_body(d, **kwargs)\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/index/mailboxes.py", "content": "from __future__ import print_function\nimport email.parser\nimport json\nimport traceback\n\nfrom mailpile.util import *\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.index.base import BaseIndex\nfrom mailpile.index.msginfo import MessageInfoConstants\nfrom mailpile.index.search import SearchResultSet\nfrom mailpile.mailutils import MBX_ID_LEN\n\n\nclass MailboxIndex(BaseIndex):\n\n FAKE_MBX_ID = ('Z' * MBX_ID_LEN)\n\n def __init__(self, config, mailbox, mbx_mid=None):\n BaseIndex.__init__(self, config)\n self.mbx_mid = mbx_mid or self.FAKE_MBX_ID\n self.mailbox = mailbox\n self.ptrset = set([])\n self.idxmap = []\n\n def get_msg_at_idx_pos_uncached(self, msg_idx_pos):\n msg_ptr = self.idxmap[msg_idx_pos]\n msg_raw = self.mailbox.get_file_by_ptr(msg_ptr)\n message = email.parser.Parser().parse(msg_raw, True)\n return self._message_to_msg_info(msg_idx_pos, msg_ptr, message)\n\n def open_mailbox_by_ptr(self, msg_ptr):\n if msg_ptr[:MBX_ID_LEN] == self.mbx_mid:\n return self.mailbox\n else:\n return BaseIndex.open_mailbox_by_ptr(self, msg_ptr)\n\n def _update_keymap(self):\n try:\n mailbox_keys = self.mailbox.keys()\n mailbox_ptrs = [self.mailbox.get_msg_ptr(self.mbx_mid, i)\n for i in mailbox_keys]\n for ptr in mailbox_ptrs:\n if ptr not in self.ptrset:\n self.idxmap.append(ptr)\n self.ptrset = set(mailbox_ptrs)\n except:\n traceback.print_exc()\n\n def search(self, session, terms, context=None):\n if not terms or terms == ['all:mail']:\n self._update_keymap()\n result = [i for i, ptr in enumerate(self.idxmap)\n if ptr in self.ptrset]\n result.reverse()\n else:\n print('FIXME! %s: search %s' % (self, terms))\n result = []\n return SearchResultSet(self, terms, result, [])\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/index/msginfo.py", "content": "from __future__ import print_function\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nclass MessageInfoConstants(object):\n MSG_MID = 0\n MSG_PTRS = 1\n MSG_ID = 2\n MSG_DATE = 3\n MSG_FROM = 4\n MSG_TO = 5\n MSG_CC = 6\n MSG_KB = 7\n MSG_SUBJECT = 8\n MSG_BODY = 9\n MSG_TAGS = 10\n MSG_REPLIES = 11\n MSG_THREAD_MID = 12\n\n MSG_FIELDS_V1 = 11\n MSG_FIELDS_V2 = 13\n\n MSG_BODY_LAZY = '{L}'\n MSG_BODY_GHOST = '{G}'\n MSG_BODY_DELETED = '{D}'\n MSG_BODY_UNSCANNED = (MSG_BODY_LAZY, MSG_BODY_GHOST)\n MSG_BODY_MAGIC = (MSG_BODY_LAZY, MSG_BODY_GHOST, MSG_BODY_DELETED)\n\n BOGUS_METADATA = [None, '', None, '0', '(no sender)', '', '', '0',\n '(not in index)', '', '', '', '-1']\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/index/search.py", "content": "from __future__ import print_function\nimport time\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nclass SearchResultSet:\n \"\"\"\n Search results!\n \"\"\"\n def __init__(self, idx, terms, results, exclude):\n self.terms = set(terms)\n self._index = idx\n self.set_results(results, exclude)\n\n def set_results(self, results, exclude):\n self._results = {\n 'raw': set(results),\n 'excluded': set(exclude) & set(results)\n }\n return self\n\n def __len__(self):\n return len(self._results.get('raw', []))\n\n def as_set(self, order='raw'):\n return self._results[order] - self._results['excluded']\n\n def excluded(self):\n return self._results['excluded']\n\n\nSEARCH_RESULT_CACHE = {}\n\n\nclass CachedSearchResultSet(SearchResultSet):\n \"\"\"\n Cached search result.\n \"\"\"\n def __init__(self, idx, terms):\n global SEARCH_RESULT_CACHE\n self.terms = set(terms)\n self._index = idx\n self._results = SEARCH_RESULT_CACHE.get(self._skey(), {})\n self._results['_last_used'] = time.time()\n\n def _skey(self):\n return ' '.join(self.terms)\n\n def set_results(self, *args):\n global SEARCH_RESULT_CACHE\n SearchResultSet.set_results(self, *args)\n SEARCH_RESULT_CACHE[self._skey()] = self._results\n self._results['_last_used'] = time.time()\n return self\n\n @classmethod\n def DropCaches(cls, msg_idxs=None, tags=None):\n # FIXME: Make this more granular\n global SEARCH_RESULT_CACHE\n SEARCH_RESULT_CACHE = {}\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mail_source/__init__.py", "content": "import datetime\nimport os\nimport random\nimport re\nimport thread\nimport threading\nimport traceback\nimport time\n\nimport mailpile.util\nimport mailpile.vfs\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import *\nfrom mailpile.mailutils import FormatMbxId\nfrom mailpile.util import *\nfrom mailpile.vfs import vfs, FilePath, MailpileVfsBase\n\n\n__all__ = ['local', 'imap', 'pop3']\n\n\nclass BaseMailSource(threading.Thread):\n \"\"\"\n MailSources take care of managing a group of mailboxes, synchronizing\n the source with Mailpile's local metadata and/or caches.\n \"\"\"\n DEFAULT_JITTER = 15 # Fudge factor to tame thundering herds\n SAVE_STATE_INTERVAL = 3600 # How frequently we pickle our state\n INTERNAL_ERROR_SLEEP = 900 # Pause time on error, in seconds\n RESCAN_BATCH_SIZE = 200 # Index at most this many new e-mails at once\n MAX_PATHS = 2000 # Limit how many directories we scan at once\n\n # This is a helper for the events.\n __classname__ = 'mailpile.mail_source.BaseMailSource'\n\n\n class MailSourceVfs(MailpileVfsBase):\n \"\"\"Generic VFS layer for this mail source.\"\"\"\n def __init__(self, config, source, *args, **kwargs):\n MailpileVfsBase.__init__(self, *args, **kwargs)\n self.config = config\n self.source = source\n self.root = FilePath('/src:%s' % self.source.my_config._key)\n\n def _get_mbox_id(self, path):\n return path[len(self.root.raw_fp)+1:]\n\n def Handles(self, path):\n path = FilePath(path)\n return (self.root == path or\n path.raw_fp.startswith(self.root.raw_fp))\n\n def glob_(self, *args, **kwargs):\n return self.listdir_(*args, **kwargs)\n\n def listdir_(self, where, **kwargs):\n return [m for m in self.source.my_config.mailbox.keys()]\n\n def open_(self, fp, *args, **kwargs):\n raise IOError('Cannot open Mail Source entries (yet)')\n\n def abspath_(self, path):\n if not path.startswith(self.root.raw_fp):\n path = self.root.join(path).raw_fp\n if path == self.root:\n return path\n try:\n mbox_id = self._get_mbox_id(path)\n path = self.config.sys.mailbox[mbox_id]\n if path.startswith('src:'):\n return '/%s' % path\n return path\n except (ValueError, KeyError, IndexError):\n raise OSError('Not found: %s' % path)\n\n def isdir_(self, fp):\n return (self.root == fp)\n\n def ismailsource_(self, fp):\n return (self.root == fp)\n\n def mailbox_type_(self, fp, config):\n return False if (fp == self.root) else 'source' # Fixme\n\n def getsize_(self, path):\n return None\n\n def display_name_(self, path, config):\n if (self.root == path):\n return (self.source.my_config.name or\n self.source.my_config._key)\n try:\n mbox_id = self._get_mbox_id(path)\n return self.source.my_config.mailbox[mbox_id].name\n except (ValueError, KeyError, IndexError):\n raise OSError('Not found: %s' % path)\n\n def exists_(self, fp):\n return ((self.root == fp) or\n (fp[len(self.root)+1:] in self.source.my_config.mailbox))\n\n\n def __init__(self, session, my_config):\n threading.Thread.__init__(self)\n self.daemon = mailpile.util.TESTING\n self._lock = MSrcRLock()\n self.my_config = my_config\n self.name = my_config.name\n self.session = session\n self.alive = None\n self.event = None\n self.jitter = self.DEFAULT_JITTER\n self._state = 'Idle'\n self._sleeping = None\n self._interrupt = None\n self._rescanning = False\n self._rescan_waiters = []\n self._rescan_forced = False\n self._loop_count = 0\n self._last_rescan_count = (0, 0)\n self._last_rescan_completed = False\n self._last_rescan_failed = False\n self._last_saved = time.time() # Saving right away would be silly\n ms_vfs = self.MailSourceVfs(session.config, self)\n mailpile.vfs.register_handler(5000, ms_vfs)\n\n def __str__(self):\n rv = ': '.join([threading.Thread.__str__(self), self._state])\n if self._sleeping > 0:\n rv += '(%s)' % self._sleeping\n return rv\n\n def _pfn(self):\n return 'mail-source.%s' % self.my_config._key\n\n def _load_state(self):\n with self._lock:\n config, my_config = self.session.config, self.my_config\n events = list(config.event_log.incomplete(source=self,\n data_id=my_config._key))\n if events:\n self.event = events[0]\n if my_config.enabled:\n self.event.message = _('Starting up')\n else:\n self.event.message = _('Disabled')\n else:\n self.event = config.event_log.log(\n source=self,\n flags=Event.RUNNING,\n message=_('Starting up'),\n data={'id': my_config._key})\n self.event.data['name'] = my_config.name or _('Mail Source')\n if 'counters' not in self.event.data:\n self.event.data['counters'] = {}\n for c in ('copied_messages',\n 'indexed_messages',\n 'unknown_policies'):\n if c not in self.event.data['counters']:\n self.event.data['counters'][c] = 0\n\n def _save_state(self):\n self.session.config.event_log.log_event(self.event)\n\n def _save_config(self):\n self.session.config.save_worker.add_unique_task(\n self.session, 'Save config', self.session.config.save)\n\n def _log_status(self, message, clear_errors=False):\n # If the user renames our parent_tag, we assume the name change too.\n self.update_name_to_match_tag()\n if clear_errors:\n err = self.event.data.get('connection', {}).get('error', [False])\n if err[0]:\n err[:] = [False, _('Nothing is wrong')]\n self.event.message = message\n self.session.config.event_log.log_event(self.event)\n self.session.ui.mark(message)\n if 'sources' in self.session.config.sys.debug:\n self.session.ui.debug('%s: %s' % (self, message))\n\n def open(self):\n \"\"\"Open mailboxes or connect to the remote mail source.\"\"\"\n raise NotImplemented('Please override open in %s' % self)\n\n def close(self):\n \"\"\"Close mailboxes or disconnect from the remote mail source.\"\"\"\n raise NotImplemented('Please override open in %s' % self)\n\n def _has_mailbox_changed(self, mbx, state):\n \"\"\"For the default sync_mail routine, report if mailbox changed.\"\"\"\n raise NotImplemented('Please override _has_mailbox_changed in %s'\n % self)\n\n def _mark_mailbox_rescanned(self, mbx, state):\n \"\"\"For the default sync_mail routine, note mailbox was rescanned.\"\"\"\n raise NotImplemented('Please override _mark_mailbox_rescanned in %s'\n % self)\n\n def _path(self, mbx):\n if mbx.path.startswith('@'):\n return self.session.config.sys.mailbox[mbx.path[1:]]\n else:\n return mbx.path\n\n def _check_interrupt(self, log=True, clear=True):\n if not self._interrupt:\n full_path = self.session.config.need_more_disk_space()\n if full_path is not None:\n self._interrupt = _('Insufficient free space in %s'\n ) % full_path\n if (mailpile.util.QUITTING or\n self._interrupt or\n not self.my_config.enabled):\n if log:\n self._log_status(_('Interrupted: %s')\n % (self._interrupt or _('Shutting down')),\n clear_errors=(not self._interrupt))\n if clear:\n self._interrupt = None\n return True\n else:\n return False\n\n def _mailbox_sort_key(self, m):\n return md5_hex(str(self._loop_count), m.name)\n\n def _sorted_mailboxes(self):\n mailboxes = self.my_config.mailbox.values()\n mailboxes.sort(key=lambda m: (\n 'inbox' in m.name.lower() and 1 or 2,\n 'sent' in m.name.lower() and 1 or 2,\n 'spam' in m.name.lower() and 1 or 2, # For training filters!\n '[Gmail]' in m.name and 2 or 1, # This goes last...\n self._mailbox_sort_key(m)))\n return mailboxes\n\n def _policy(self, mbx_cfg):\n policy = mbx_cfg.policy\n if policy == 'inherit':\n return self.my_config.discovery.policy\n return policy\n\n def update_name_to_match_tag(self):\n parent_tag_id = self.my_config.discovery.parent_tag\n if parent_tag_id and parent_tag_id != '!CREATE':\n tag = self.session.config.get_tag(parent_tag_id)\n if tag and self.name != tag.name:\n self.name = self.my_config.name = tag.name\n if self.event:\n self.event.data['name'] = self.name\n\n def sync_mail(self):\n \"\"\"Iterates through all the mailboxes and scans if necessary.\"\"\"\n config = self.session.config\n rescanned = messages = errors = 0\n self._last_rescan_count = (0, 0)\n self._last_rescan_completed = False\n self._last_rescan_failed = False\n self._interrupt = None\n batch = min(self._loop_count * 20, self.RESCAN_BATCH_SIZE)\n errors = rescanned = 0\n all_completed = True\n ostate = self._state\n\n if not self._check_interrupt(clear=False):\n self._state = 'Waiting... (disco)'\n discovered = self.discover_mailboxes()\n else:\n discovered = 0\n\n plan = self._sorted_mailboxes()\n self.event.data['plan'] = [[m._key, _('Pending'), m.name] for m in plan]\n event_plan = dict((mp[0], mp) for mp in self.event.data['plan'])\n if plan and random.randint(0, 20) == 1:\n random_plan = [m._key for m in random.sample(plan, 1)]\n else:\n random_plan = []\n\n for mbx_cfg in plan:\n if not self._rescan_forced:\n play_nice_with_threads(weak=True)\n\n if self._check_interrupt(clear=False):\n all_completed = False\n break\n try:\n with self._lock:\n mbx_key = FormatMbxId(mbx_cfg._key)\n path = self._path(mbx_cfg)\n policy = self._policy(mbx_cfg)\n if (path in ('/dev/null', '', None)\n or policy in ('ignore', 'unknown')):\n event_plan[mbx_cfg._key][1] = _('Skipped')\n continue\n\n # Generally speaking, we only rescan if a mailbox looks like\n # it has changed. However, every once in a while (see logic\n # around random_mailboxes above) we check anyway just in case\n # looks are deceiving.\n state = {}\n if batch < 1:\n event_plan[mbx_cfg._key][1] = _('Postponed')\n\n elif (self._has_mailbox_changed(mbx_cfg, state) or\n self._rescan_forced or\n mbx_cfg.local == '!CREATE' or\n mbx_cfg._key in random_plan):\n event_plan[mbx_cfg._key][1] = _('Working ...')\n\n this_batch = max(5, int(0.7 * batch))\n self._state = 'Waiting... (rescan)'\n if self._check_interrupt(clear=False):\n all_completed = False\n break\n count = self.rescan_mailbox(mbx_key, mbx_cfg, path,\n stop_after=this_batch)\n\n if count >= 0:\n self.event.data['counters'\n ]['indexed_messages'] += count\n batch -= count\n this_batch -= count\n messages += count\n complete = ((count == 0 or this_batch > 0) and\n not self._interrupt and\n not mailpile.util.QUITTING)\n if complete:\n rescanned += 1\n\n # If there was a copy, check if it completed\n cstate = self.event.data.get('copying') or {}\n if not cstate.get('complete', True):\n complete = False\n\n # If there was a rescan, check if it completed\n rstate = self.event.data.get('rescan') or {}\n if not rstate.get('complete', True):\n complete = False\n\n # OK, everything looks complete, mark it!\n if complete:\n event_plan[mbx_cfg._key][1] = _('Completed')\n self._mark_mailbox_rescanned(mbx_cfg, state)\n else:\n event_plan[mbx_cfg._key][1] = _('Indexed %d'\n ) % count\n all_completed = False\n if count == 0 and ('sources' in config.sys.debug):\n time.sleep(60)\n else:\n event_plan[mbx_cfg._key][1] = _('Failed')\n self._last_rescan_failed = True\n all_completed = False\n errors += 1\n\n else:\n event_plan[mbx_cfg._key][1] = _('Unchanged')\n\n except (NoSuchMailboxError, IOError, OSError) as e:\n event_plan[mbx_cfg._key][1] = '%s: %s' % (_('Error'), e)\n self._last_rescan_failed = True\n errors += 1\n except Exception as e:\n event_plan[mbx_cfg._key][1] = '%s: %s' % (\n _('Internal error'), e)\n self._rescan_forced = False\n self._last_rescan_failed = True\n self._log_status(_('Internal error'))\n raise\n\n self._last_rescan_completed = all_completed\n\n self._state = 'Done'\n status = []\n if discovered > 0:\n status.append(_('Discovered %d mailboxes') % discovered)\n self._last_rescan_completed = False\n if rescanned > 0:\n status.append(_('Processed %d mailboxes') % rescanned)\n if errors:\n status.append(_('Failed to process %d') % errors)\n if not status:\n status.append(_('No new mail at %s'\n ) % datetime.datetime.today().strftime('%H:%M'))\n\n self._rescan_forced = False\n self._log_status(', '.join(status))\n self._last_rescan_count = (messages, rescanned)\n self._state = ostate\n return rescanned\n\n def _jitter(self, seconds):\n return seconds + random.randint(0, self.jitter)\n\n def _sleeping_is_ok(self, slept):\n return True\n\n def _sleep(self, seconds):\n enabled = self.my_config.enabled\n if self._rescan_forced:\n seconds = 0\n else:\n play_nice_with_threads()\n if 'sources' in self.session.config.sys.debug:\n self.session.ui.debug('Sleeping up to %d seconds...' % seconds)\n if self._sleeping is None:\n self._sleeping = seconds\n while (self.alive and\n (self._sleeping > 0) and\n self._sleeping_is_ok(seconds - self._sleeping) and\n (enabled == self.my_config.enabled) and\n not mailpile.util.QUITTING):\n time.sleep(max(0, min(1, self._sleeping)))\n self._sleeping -= 1\n self._sleeping = None\n return (self.alive and not mailpile.util.QUITTING)\n\n def _existing_mailboxes(self):\n return set(self.session.config.sys.mailbox +\n [mbx_cfg.local\n for mbx_cfg in self.my_config.mailbox.values()\n if mbx_cfg.local])\n\n def _update_unknown_state(self):\n have_unknown = 0\n for mailbox in self.my_config.mailbox.values():\n if mailbox.policy == 'unknown':\n have_unknown += 1\n self.event.data['counters']['unknown_policies'] = have_unknown\n self.event.data['have_unknown'] = (have_unknown > 0)\n\n def reset_event_discovery_state(self):\n for k in ('discovery_error', 'discovery_limit', 'discovery_state'):\n if k in self.event.data:\n del self.event.data[k]\n\n def set_event_discovery_state(self, state=None, error=None, status=None):\n self.event.data['discovery_limit'] = (\n self.my_config.discovery.max_mailboxes)\n if state is not None:\n self.event.data['discovery_state'] = state\n if error is not None:\n self.event.data['discovery_error'] = error\n if status is not None:\n self._log_status(status)\n\n def on_event_discovery_starting(self):\n ostate, self._state = self._state, 'Discovery'\n self.reset_event_discovery_state()\n self.set_event_discovery_state(\n 'scanning', status=_('Checking for new mailboxes'))\n return ostate\n\n def on_event_discovery_toomany(self):\n self.set_event_discovery_state(\n error='toomany',\n status=_('Too many mailboxes found! Raise limits to continue.'))\n\n old_limit = self.my_config.discovery.max_mailboxes\n for i in range(0, 15):\n if old_limit == self.my_config.discovery.max_mailboxes:\n self._sleep(2)\n else:\n return True\n return False\n\n def on_event_discovery_done(self, ostate):\n self.set_event_discovery_state('done')\n self._state = ostate\n\n def discover_mailboxes(self, paths=None):\n config = self.session.config\n ostate = self.on_event_discovery_starting()\n try:\n existing = self._existing_mailboxes()\n max_mailboxes = self.my_config.discovery.max_mailboxes\n max_mailboxes -= len(self.my_config.mailbox)\n adding = []\n paths = [(p.encode('utf-8') if isinstance(p, unicode) else p)\n for p in (paths or self.my_config.discovery.paths)]\n paths.sort()\n while paths:\n raw_fn = paths.pop(0)\n if 'sources' in config.sys.debug:\n self.session.ui.mark(_('Checking for new mailboxes in %s'\n ) % raw_fn.decode('utf-8'))\n\n fn = os.path.normpath(os.path.expanduser(raw_fn))\n fn = os.path.abspath(fn)\n if not os.path.exists(fn):\n continue\n\n is_mailbox = False\n if (raw_fn not in existing and\n fn not in existing and\n fn not in adding):\n if self.is_mailbox(fn):\n adding.append(fn)\n is_mailbox = True\n if len(adding) and (len(adding) > max_mailboxes):\n break\n\n if os.path.isdir(fn):\n try:\n max_paths = self.MAX_PATHS - len(paths)\n subdirs = [f for f in os.listdir(fn)\n if f not in ('.', '..')]\n\n if len(subdirs) > (max_paths/2):\n # If we are hitting our limits, randomize.\n random.shuffle(subdirs)\n else:\n # Otherwise, do things in an orderly fashion.\n subdirs.sort()\n\n for f in subdirs[:max_paths/2]:\n nfn = os.path.join(fn, f)\n if is_mailbox and f in ('cur', 'new', 'tmp'):\n pass # Skip Maildir special directories\n elif (len(paths) <= self.MAX_PATHS and\n os.path.isdir(nfn)):\n paths.append(nfn)\n elif self.is_mailbox(nfn):\n paths.append(nfn)\n play_nice_with_threads(weak=True)\n except OSError:\n pass\n\n # This may have been a bit of work, take a break.\n play_nice_with_threads()\n\n if len(adding) and (len(adding) > max_mailboxes):\n break\n\n if len(adding) and (len(adding) > max_mailboxes):\n if self.on_event_discovery_toomany():\n return self.discover_mailboxes(paths=paths)\n adding = adding[:max(0, max_mailboxes)]\n\n if adding:\n self.set_event_discovery_state('adding')\n play_nice_with_threads()\n new = {}\n for path in adding:\n new[config.sys.mailbox.append(path)] = path\n for mailbox_idx in new.keys():\n mbx_cfg = self.take_over_mailbox(mailbox_idx, save=False)\n if self._policy(mbx_cfg) != 'unknown':\n del new[mailbox_idx]\n\n self._save_config()\n\n return len(adding)\n except:\n if config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n raise\n finally:\n self.on_event_discovery_done(ostate)\n\n def _default_policy(self, mbx_cfg):\n return 'inherit'\n\n def take_over_mailbox(self, mailbox_idx,\n policy=None, create_local=None, save=True,\n guess_tags=None, apply_tags=None):\n config = self.session.config\n disco_cfg = self.my_config.discovery # Stayin' alive! Stayin' alive!\n with self._lock:\n mailbox_idx = FormatMbxId(mailbox_idx)\n self.my_config.mailbox[mailbox_idx] = {\n 'path': '@%s' % mailbox_idx,\n 'policy': policy or 'inherit',\n 'process_new': disco_cfg.process_new,\n 'local': '!CREATE' if create_local else '',\n }\n mbx_cfg = self.my_config.mailbox[mailbox_idx]\n mbx_cfg.apply_tags.extend(disco_cfg.apply_tags)\n if apply_tags:\n mbx_cfg.apply_tags.extend(t for t in apply_tags if t)\n mbx_cfg.policy = policy or self._default_policy(mbx_cfg)\n mbx_cfg.name = self._mailbox_name(self._path(mbx_cfg))\n if guess_tags is None:\n guess_tags = disco_cfg.guess_tags\n if guess_tags:\n self._guess_tags(mbx_cfg)\n self._create_primary_tag(mbx_cfg, save=False)\n self._create_local_mailbox(mbx_cfg, save=False)\n if save:\n self._save_config()\n return mbx_cfg\n\n def _guess_tags(self, mbx_cfg):\n if not mbx_cfg.name:\n return\n mbx_cfg.apply_tags = sorted(list(\n set(mbx_cfg.apply_tags) |\n self.session.config.guess_tags(mbx_cfg.name)))\n\n def _strip_file_extension(self, path):\n return path.rsplit('.', 1)[0]\n\n def _mailbox_path_split(self, path):\n return ('/' in path) and path.split('/') or path.split('\\\\')\n\n def _mailbox_name(self, path):\n return self._mailbox_path_split(path)[-1]\n\n def _create_local_mailbox(self, mbx_cfg, save=True):\n config = self.session.config\n disco_cfg = self.my_config.discovery\n\n if mbx_cfg.local and mbx_cfg.local != '!CREATE':\n if not vfs.exists(mbx_cfg.local):\n config.flush_mbox_cache(self.session)\n path, wervd = config.create_local_mailstore(self.session,\n name=mbx_cfg.local)\n wervd.is_local = mbx_cfg._key\n mbx_cfg.local = path\n if save:\n self._save_config()\n\n elif mbx_cfg.local == '!CREATE' or disco_cfg.local_copy:\n config.flush_mbox_cache(self.session)\n path, wervd = config.create_local_mailstore(self.session)\n wervd.is_local = mbx_cfg._key\n mbx_cfg.local = path\n if save:\n self._save_config()\n\n return mbx_cfg\n\n def _create_parent_tag(self, save=True):\n disco_cfg = self.my_config.discovery\n if disco_cfg.parent_tag:\n if disco_cfg.parent_tag == '!CREATE':\n name = (self.my_config.name or\n (self.my_config.username or '').split('@')[-1] or\n (disco_cfg.paths and\n os.path.basename(disco_cfg.paths[0])) or\n self.my_config._key)\n if len(name) < 4:\n name = _('Mail: %s') % name\n disco_cfg.parent_tag = name\n if disco_cfg.parent_tag not in self.session.config.tags.keys():\n from mailpile.plugins.tags import Slugify\n disco_cfg.parent_tag = self._create_tag(\n disco_cfg.parent_tag,\n use_existing=False,\n icon='icon-mailsource',\n slug=Slugify(\n self.my_config.name, tags=self.session.config.tags),\n unique=False)\n if save:\n self._save_config()\n return disco_cfg.parent_tag\n else:\n return None\n\n def _create_primary_tag(self, mbx_cfg, save=True):\n config = self.session.config\n if mbx_cfg.primary_tag and (mbx_cfg.primary_tag in config.tags):\n return\n\n # Stayin' alive! Stayin' alive!\n disco_cfg = self.my_config.discovery\n if not disco_cfg.create_tag:\n return\n\n # Make sure we have a parent tag, as that maybe useful when creating\n # tag names or the primary tag itself.\n parent = self._create_parent_tag(save=False)\n\n # We configure the primary_tag with a name, if it doesn't have\n # one already.\n if not mbx_cfg.primary_tag:\n mbx_cfg.primary_tag = self._create_tag_name(self._path(mbx_cfg))\n\n # If we have a policy for this mailbox, we really go and create\n # tags. The gap here allows the user to edit the primary_tag\n # proposal before changing the policy from 'unknown'.\n if self._policy(mbx_cfg) != 'unknown':\n try:\n mbx_cfg.primary_tag = self._create_tag(\n mbx_cfg.primary_tag,\n use_existing=False,\n visible=disco_cfg.visible_tags,\n unique=False,\n parent=parent)\n except (ValueError, IndexError):\n self.session.ui.debug(traceback.format_exc())\n\n if save:\n self._save_config()\n\n BORING_FOLDER_RE = re.compile('(?i)^(home|mail|data|user\\S*|[^[:alpha:]]+)$', re.UNICODE)\n TAGNAME_STRIP_RE = re.compile('[{}\\\\[\\\\]]', re.UNICODE)\n\n def _path_to_tagname(self, path): # -> tag name\n \"\"\"This converts a path to a tag name.\"\"\"\n parts = self._mailbox_path_split(path)\n parts = [p for p in parts if not re.match(self.BORING_FOLDER_RE, p)]\n if not parts:\n return _('Unnamed')\n tagname = self._strip_file_extension(parts.pop(-1))\n while tagname[:1] == '.':\n tagname = tagname[1:]\n return re.sub(self.TAGNAME_STRIP_RE, '', tagname.replace('_', ' '))\n\n def _unique_tag_name(self, tagname): # -> unused tag name\n \"\"\"Make sure a tagname really is unused, unless we have a parent\"\"\"\n if self.my_config.discovery.parent_tag:\n return tagname\n tagnameN, count = tagname, 2\n while self.session.config.get_tags(tagnameN):\n tagnameN = '%s (%s)' % (tagname, count)\n count += 1\n return tagnameN\n\n def _create_tag_name(self, path): # -> unique tag name\n \"\"\"Convert a path to a unique tag name.\"\"\"\n return self._unique_tag_name(self._path_to_tagname(path))\n\n def _create_tag(self, tag_name_or_id,\n use_existing=True,\n unique=False,\n label=False,\n visible=True,\n slug=None,\n icon=None,\n parent=None): # -> tag ID\n if tag_name_or_id in self.session.config.tags:\n # Short circuit if this is a tag ID for an existing tag\n return tag_name_or_id\n else:\n tag_name = tag_name_or_id\n\n tags = self.session.config.get_tags(tag_name)\n if tags and unique:\n raise ValueError('Tag name is not unique!')\n elif len(tags) == 1 and use_existing:\n tag_id = tags[0]._key\n else:\n if slug is None:\n from mailpile.plugins.tags import Slugify\n if self.my_config.name:\n slug = Slugify('/'.join([self.my_config.name, tag_name]),\n tags=self.session.config.tags)\n else:\n slug = Slugify(tag_name, tags=self.session.config.tags)\n tag_id = self.session.config.tags.append({\n 'name': tag_name,\n 'slug': slug,\n 'type': 'mailbox',\n 'parent': parent or '',\n 'label': label,\n 'flag_allow_add': False,\n 'flag_allow_del': False,\n 'icon': icon or 'icon-tag',\n 'display': 'tag' if visible else 'archive',\n })\n if parent and visible:\n self.session.config.tags[parent].display = 'tag'\n return tag_id\n\n def interrupt_rescan(self, reason):\n self._interrupt = reason or _('Aborted')\n if self._rescanning:\n self.session.config.index.interrupt = reason\n\n def _process_new(self, mbx_key, mbx_cfg, mbox,\n msg, msg_metadata_kws, msg_ts, keywords, snippet):\n # Here subclasses could use mbx_key, mbx_cfg or mbox to grab the\n # mailbox itself, in case it has metadata (like Maildir). The\n # default just looks at the Status: headers of the mail itself.\n return ProcessNew(self.session, msg, msg_metadata_kws, msg_ts,\n keywords, snippet)\n\n def _msg_key_order(self, key):\n return key\n\n def _copy_new_messages(self, mbx_key, mbx_cfg, src,\n stop_after=-1, scan_args=None, deadline=None):\n session, config = self.session, self.session.config\n self.event.data['copying'] = progress = {\n 'running': True,\n 'mailbox_id': mbx_key,\n 'copied_messages': 0,\n 'copied_bytes': 0,\n 'deleting': False,\n 'complete': False}\n scan_args = scan_args or {}\n policy = self._policy(mbx_cfg)\n count = 0\n\n def maybe_delete_from_server(loc, src):\n # Delete from source, if that's our policy.\n if policy != 'move':\n return\n\n # Messages we have downloaded are candidates for deletion.\n downloaded = list(set(src.keys()) & set(loc.source_map.keys()))\n downloaded.sort(key=self._msg_key_order)\n\n # If prefs.deletion_ratio is less than 1.0, leave the most\n # recent messages on the server (so other clients have a\n # chance to see them too).\n #\n # FIXME: This is a hack. It would be better to check the\n # timestamps of the messages and always leave on the\n # server for a fixed, configurable amount of time.\n #\n if config.prefs.deletion_ratio < 1.0:\n random_ratio = random.random() * config.prefs.deletion_ratio\n cutoff = int(max(\n random_ratio * len(downloaded),\n # Jitter. Without this, the last message never gets deleted:\n random.randint(-10, 1)))\n else:\n cutoff = len(downloaded)\n\n should = _('Should delete %d messages') % cutoff\n if 'sources' in config.sys.debug and downloaded:\n session.ui.debug(should)\n\n if config.prefs.allow_deletion:\n try:\n for i, key in enumerate(downloaded):\n if i >= cutoff:\n break\n progress['deleting'] = '%d/%d' % (i+1, cutoff)\n src.remove(key)\n src.flush()\n except:\n # Just ignore errors for now, we'll try again later.\n if 'sources' in config.sys.debug:\n session.ui.debug(traceback.format_exc())\n else:\n progress['deleting'] = '. '.join([\n _('Deletion is disabled'), should])\n\n try:\n # Lock the source mailbox while we work with it\n src.lock()\n\n with self._lock:\n loc = config.open_mailbox(session, mbx_key, prefer_local=True)\n if src == loc:\n return count\n\n # Perform housekeeping on the source_map, to make sure it does\n # not grow without bounds or misrepresent things.\n gone = []\n src_keys = set(src.keys())\n loc_keys = set(loc.keys())\n for key, val in loc.source_map.iteritems():\n if (val not in loc_keys) or (key not in src_keys):\n gone.append(key)\n for key in gone:\n del loc.source_map[key]\n\n # Figure out what actually needs to be downloaded, log it\n keys = list(src_keys - set(loc.source_map.keys()))\n keys.sort(key=self._msg_key_order)\n progress.update({\n 'total': len(src_keys),\n 'total_local': len(loc_keys),\n 'uncopied': len(keys),\n 'batch_size': stop_after if (stop_after > 0) else len(keys)})\n\n # Go download!\n key_errors = []\n for key in reversed(keys):\n if self._check_interrupt(log=False, clear=False):\n progress['interrupted'] = True\n return count\n\n session.ui.mark(_('Copying message: %s') % key)\n progress['copying_src_id'] = key\n try:\n mkws = src.get_metadata_keywords(key)\n data = src.get_bytes(key)\n except KeyError:\n progress['key_errors'] = key_errors\n key_errors.append(key)\n # Ignore, in case this is a problem with just this\n # individual message...\n continue\n\n loc_key = loc.add_from_source(key, mkws, data)\n self.event.data['counters']['copied_messages'] += 1\n del progress['copying_src_id']\n progress['copied_messages'] += 1\n progress['copied_bytes'] += len(data)\n progress['uncopied'] -= 1\n count += 1\n\n # This forks off a scan job to index the message\n config.index.scan_one_message(\n session, mbx_key, loc, loc_key,\n wait=False, msg_data=data, msg_metadata_kws=mkws,\n **scan_args)\n\n stop_after -= 1\n if (stop_after == 0) or (deadline and time.time() > deadline):\n maybe_delete_from_server(loc, src)\n progress['stopped'] = True\n return count\n progress['complete'] = True\n\n except IOError:\n # These just abort the download/read, which we're going to just\n # take in stride for now.\n if 'sources' in config.sys.debug:\n session.ui.debug(traceback.format_exc())\n progress['ioerror'] = True\n except:\n if 'sources' in config.sys.debug:\n session.ui.debug(traceback.format_exc())\n progress['raised'] = True\n raise\n finally:\n progress['running'] = False\n src.unlock()\n\n maybe_delete_from_server(loc, src)\n return count\n\n def rescan_mailbox(self, mbx_key, mbx_cfg, path, stop_after=None):\n session, config = self.session, self.session.config\n\n with self._lock:\n if self._rescanning:\n return -1\n self._rescanning = True\n\n mailboxes = min(1, len([m for m in self.my_config.mailbox.values()\n if self._policy(m) not in ('ignore',\n 'unknown')]))\n try:\n ostate = self._state # Set this in case locking fails\n with self._lock:\n new_state = 'Rescan(%s, %s)' % (mbx_key, stop_after)\n ostate, self._state = self._state, new_state\n\n apply_tags = mbx_cfg.apply_tags[:]\n\n parent = self._create_parent_tag(save=True)\n if parent:\n tid = config.get_tag_id(parent)\n if tid:\n apply_tags.append(tid)\n\n self._create_primary_tag(mbx_cfg)\n if mbx_cfg.primary_tag:\n tid = config.get_tag_id(mbx_cfg.primary_tag)\n if tid:\n apply_tags.append(tid)\n\n with self._lock:\n mbox = config.open_mailbox(session, mbx_key,\n prefer_local=False)\n def process_new(msg, msg_metadata_kws, msg_ts, keywords, snippet):\n return self._process_new(mbx_key, mbx_cfg, mbox,\n msg, msg_metadata_kws, msg_ts,\n keywords, snippet)\n scan_mailbox_args = {\n 'process_new': (process_new if mbx_cfg.process_new else False),\n 'apply_tags': (apply_tags or []),\n 'stop_after': stop_after,\n 'event': self.event}\n copied = count = 0\n\n if mbx_cfg.local or self.my_config.discovery.local_copy:\n # Note: We copy fewer messages than the batch allows for,\n # because we might have been aborted on an earlier run and\n # the rescan may need to catch up.\n self._create_local_mailbox(mbx_cfg)\n max_copy = max(min(stop_after, 5), int(0.8 * stop_after))\n self._state = '%s: %s' % (new_state, _('Copying'))\n self._log_status(_('Copying up to %d e-mails from %s'\n ) % (max_copy, self._mailbox_name(path)))\n copied = self._copy_new_messages(mbx_key, mbx_cfg, mbox,\n stop_after=max_copy,\n scan_args=scan_mailbox_args)\n count += copied\n\n if self._check_interrupt(clear=False):\n if 'rescan' in self.event.data:\n self.event.data['rescan']['running'] = False\n return count\n\n self._state = '%s: %s' % (new_state, _('Working'))\n self._log_status(_('Updating search engine for %s'\n ) % self._mailbox_name(path))\n # Wait for background message scans to complete...\n config.scan_worker.do(session, 'Wait:%s' % path, lambda: 1)\n\n if 'rescans' in self.event.data:\n self.event.data['rescans'][:-mailboxes] = []\n\n return count + config.index.scan_mailbox(session,\n mbx_key,\n mbx_cfg.local or path,\n config.open_mailbox,\n force=self._rescan_forced,\n **scan_mailbox_args)\n except ValueError:\n session.ui.debug(traceback.format_exc())\n return -1\n finally:\n self._state = ostate\n self._rescanning = False\n\n def open_mailbox(self, mbx_id, fn):\n # This allows mail sources to override the default mailbox\n # opening mechanism. Returning false respectfully declines.\n return None\n\n def is_mailbox(self, fn):\n return False\n\n def _summarize_auth(self):\n return sha1b64(self.my_config.auth_type, '-',\n self.my_config.username, '-',\n self.my_config.password)\n\n def run(self):\n play_nice(18) # Reduce priority quite a lot\n\n with self.session.config.index_check:\n self.alive = True\n\n self._load_state()\n _original_session = self.session\n\n def sleeptime():\n if not self.my_config.enabled:\n return 24 * 3600\n elif self._last_rescan_completed or self._last_rescan_failed:\n return self.my_config.interval\n else:\n return 1\n\n def have_invalid_auth():\n conn_err = self.event.data.get('connection', {}).get('error')\n if conn_err and conn_err[0] in ('oauth2', 'auth'):\n if ((self._loop_count % 100 != 0)\n and self._summarize_auth() == conn_err[-1]):\n self.session.ui.debug('Auth unchanged, doing nothing')\n return True\n else:\n self._log_status(_('Checking new credentials'),\n clear_errors=True)\n return False\n\n self._loop_count = 0\n while self._loop_count == 0 or self._sleep(self._jitter(sleeptime())):\n self.event.data['enabled'] = self.my_config.enabled\n self.event.data['profile_id'] = self.my_config.profile\n if self.my_config.enabled:\n self.event.flags = Event.RUNNING\n self._loop_count += 1\n else:\n if self._loop_count > 1:\n self._log_status(_('Disabled'), clear_errors=True)\n self._loop_count = 1\n self.close()\n continue\n\n self.name = self.my_config.name # In case the config changes\n self._update_unknown_state()\n if not self.session.config.index:\n continue\n\n if have_invalid_auth():\n continue\n\n waiters, self._rescan_waiters = self._rescan_waiters, []\n for b, e, s in waiters:\n try:\n b.release()\n except thread.error:\n pass\n if s:\n self.session = s\n try:\n if 'traceback' in self.event.data:\n del self.event.data['traceback']\n if self.open():\n self.sync_mail()\n else:\n self._log_conn_errors()\n\n next_save_time = self._last_saved + self.SAVE_STATE_INTERVAL\n if self.alive and time.time() >= next_save_time:\n self._save_state()\n self._check_keepalive()\n elif self._last_rescan_completed:\n self._check_keepalive()\n except:\n self.event.data['traceback'] = traceback.format_exc()\n self.session.ui.debug(self.event.data['traceback'])\n self._log_status(_('Internal error! Sleeping...'))\n self._sleep(self.INTERNAL_ERROR_SLEEP)\n finally:\n for b, e, s in waiters:\n try:\n e.release()\n except thread.error:\n pass\n self.session = _original_session\n self._update_unknown_state()\n self.close()\n self._log_status(_('Shutdown'), clear_errors=True)\n self._save_state()\n\n def _check_keepalive(self):\n if not self.my_config.keepalive:\n self.close()\n\n def _log_conn_errors(self):\n if 'connection' in self.event.data:\n cinfo = self.event.data['connection']\n if not cinfo.get('live'):\n err_msg = cinfo.get('error', [None, None])[1]\n if err_msg:\n self._log_status(err_msg)\n\n def wake_up(self):\n if 'wakeup' in self.session.config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_stack())\n self._sleeping = -1\n\n def notify_config_changed(self):\n # FIXME: It would be nice to check if the changes apply to us and\n # stay asleep otherwise.\n self.wake_up()\n\n def rescan_now(self, session=None, started_callback=None):\n if not self.my_config.enabled:\n return (0, 0)\n\n begin, end = MSrcLock(), MSrcLock()\n for l in (begin, end):\n l.acquire()\n try:\n self._rescan_waiters.append((begin, end, session))\n self._rescan_forced = True\n self._interrupt = 'Rescan forced'\n self.wake_up()\n while not begin.acquire(False):\n time.sleep(1)\n if mailpile.util.QUITTING:\n return self._last_rescan_count\n if started_callback:\n started_callback()\n while not end.acquire(False):\n time.sleep(1)\n if mailpile.util.QUITTING:\n return self._last_rescan_count\n return self._last_rescan_count\n except KeyboardInterrupt:\n self.interrupt_rescan(_('User aborted'))\n raise\n finally:\n for l in (begin, end):\n try:\n l.release()\n except thread.error:\n pass\n\n def quit(self, join=False):\n self.interrupt_rescan(_('Shutdown'))\n self.alive = False\n self.wake_up()\n if join and self.isAlive():\n self.join()\n\n\ndef ProcessNew(session, msg, msg_metadata_kws, msg_ts, keywords, snippet):\n if False and ('dsn:has' in keywords or 'mdn:has' in keywords):\n # FIXME: This is a delivery notfication of some sort!\n # TODO: Figure out what it is telling us, do not mark as \"new\".\n return False\n\n if ('s:maildir' in msg_metadata_kws # Seen=read, maildir\n or 'r:maildir' in msg_metadata_kws # Replied, maildir\n or 'r' in msg.get('status', '').lower() # Read, mbox\n or 'a' in msg.get('x-status', '').lower()): # PINE, answered\n return False\n\n keywords.update(['%s:in' % tag._key for tag in\n session.config.get_tags(type='unread')])\n return True\n\n\ndef MailSource(session, my_config):\n # FIXME: check the plugin and instanciate the right kind of mail source\n # for this config section.\n if my_config.protocol in ('mbox', 'maildir', 'local'):\n from mailpile.mail_source.local import LocalMailSource\n return LocalMailSource(session, my_config)\n elif my_config.protocol in ('imap', 'imap_ssl', 'imap_tls'):\n from mailpile.mail_source.imap import ImapMailSource\n return ImapMailSource(session, my_config)\n elif my_config.protocol in ('pop3', 'pop3_ssl'):\n from mailpile.mail_source.pop3 import Pop3MailSource\n return Pop3MailSource(session, my_config)\n raise ValueError(_('Unknown mail source protocol: %s'\n ) % my_config.protocol)\n" }, { "path": "mailpile/mail_source/imap.py", "content": "from __future__ import print_function\n# This implements our IMAP mail source. It has been tested against the\n# following IMAP implementations:\n#\n# * Google's GMail (july 2014)\n# * UW IMAPD (10.1.legacy from 2001)\n#\n#\n# IMAP resonses seen in the wild:\n#\n# GMail:\n#\n# Message flags: \\* \\Answered \\Flagged \\Draft \\Deleted \\Seen\n# $Phishing receipt-handled $NotPhishing Junk\n#\n# LIST (\\HasNoChildren) \"/\" \"Travel\"\n# LIST (\\Noselect \\HasChildren) \"/\" \"[Gmail]\"\n#\n# UW IMAPD 10.1:\n#\n# Message flags: \\* \\Answered \\Flagged \\Deleted \\Draft \\Seen\n#\n# LIST (\\NoSelect) \"/\" 17.03.2002\n# LIST (\\NoInferiors \\Marked) \"/\" in\n# LIST (\\NoInferiors \\UnMarked) \"/\" todays-junk\n#\n# Fastmail.fm:\n#\n# Message flags: \\Answered \\Flagged \\Draft \\Deleted \\Seen $X-ME-Annot-2\n# $IsMailingList $IsNotification $HasAttachment $HasTD\n#\n# LIST (\\Noinferiors \\HasNoChildren) \".\" INBOX\n# LIST (\\HasNoChildren \\Archive) \".\" Archive\n# LIST (\\HasNoChildren \\Drafts) \".\" Drafts\n# LIST (\\HasNoChildren \\Junk) \".\" \"Junk Mail\"\n# LIST (\\HasNoChildren \\Sent) \".\" \"Sent Items\"\n# LIST (\\HasNoChildren \\Trash) \".\" Trash\n#\n# Mykolab.com:\n#\n#\n#\nimport copy\nimport imaplib\nimport os\nimport re\nimport socket\nimport select\nimport ssl\nimport traceback\nimport time\nfrom imaplib import IMAP4_SSL, CRLF\nfrom mailbox import Mailbox, Message\nfrom urllib import quote, unquote\n\ntry:\n import cStringIO as StringIO\nexcept ImportError:\n import StringIO\n\nimport mailpile.mail_source.imap_utf7\nfrom mailpile.auth import IndirectPassword\nfrom mailpile.conn_brokers import Master as ConnBroker\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.index.mailboxes import MailboxIndex\nfrom mailpile.mail_source import BaseMailSource\nfrom mailpile.mail_source.imap_starttls import IMAP4\nfrom mailpile.mailutils import FormatMbxId, MBX_ID_LEN\nfrom mailpile.plugins.oauth import OAuth2\nfrom mailpile.util import *\nfrom mailpile.vfs import FilePath\n\n\n# Raise imaplib's default maximum line length to something long and\n# silly. Some versions of Python ship with this set too low for the\n# Real World (no matter what the RFCs say).\nimaplib._MAXLINE = 10 * 1024 * 1024\n\n\nIMAP_TOKEN = re.compile('(\"[^\"]*\"'\n '|[\\\\(\\\\)]'\n '|[^\\\\(\\\\)\"\\\\s]+'\n '|\\\\s+)')\n\n# These are mailbox names we avoid downloading (by default)\nBLACKLISTED_MAILBOXES = (\n 'drafts',\n 'chats',\n '[gmail]/all mail',\n '[gmail]/important',\n '[gmail]/starred',\n 'openpgp_keys')\n\n\nclass IMAP_IOError(IOError):\n pass\n\n\nclass WithaBool(object):\n def __init__(self, v): self.v = v\n def __nonzero__(self): return self.v\n def __enter__(self, *a, **kw): return self.v\n def __exit__(self, *a, **kw): return self.v\n\n\ndef _parse_imap(reply):\n \"\"\"\n This routine will parse common IMAP4 responses into Pythonic data\n structures.\n\n >>> _parse_imap(('OK', ['One (Two (Th ree)) \"Four Five\"']))\n (True, ['One', ['Two', ['Th', 'ree']], 'Four Five'])\n\n >>> _parse_imap(('BAD', ['Sorry']))\n (False, ['Sorry'])\n \"\"\"\n if not reply or len(reply) < 2:\n return False, []\n stack = []\n pdata = []\n for dline in reply[1]:\n while True:\n if isinstance(dline, (str, unicode)):\n m = IMAP_TOKEN.match(dline)\n else:\n print('WARNING: Unparsed IMAP response data: %s' % (dline,))\n m = None\n if m:\n token = m.group(0)\n dline = dline[len(token):]\n if token[:1] == '\"':\n pdata.append(token[1:-1])\n elif token[:1] == '(':\n stack.append(pdata)\n pdata.append([])\n pdata = pdata[-1]\n elif token[:1] == ')':\n pdata = stack.pop(-1)\n elif token[:1] not in (' ', '\\t', '\\n', '\\r'):\n pdata.append(token)\n else:\n break\n return (reply[0].upper() == 'OK'), pdata\n\n\nclass ImapMailboxIndex(MailboxIndex):\n pass\n\n\nclass SharedImapConn(threading.Thread):\n \"\"\"\n This is a wrapper around an imaplib.IMAP4 connection which facilitates\n sharing of the same conn between different parts of the app.\n\n If nobody is using the connection and an IDLE callback is specified,\n it will switch to IMAP IDLE mode when not otherwise in use.\n\n Callers are expected to use the \"with sharedconn as conn: ...\" syntax.\n \"\"\"\n def __init__(self, session, conn, idle_mailbox=None, idle_callback=None):\n threading.Thread.__init__(self)\n self.daemon = True\n self.session = session\n self._lock = MSrcLock()\n self._conn = conn\n self._idle_mailbox = idle_mailbox\n self._idle_callback = idle_callback\n self._can_idle = False\n self._idling = False\n self._selected = None\n\n for meth in ('append', 'add', 'authenticate', 'capability', 'fetch',\n 'noop', 'store', 'expunge', 'close',\n 'list', 'login', 'logout', 'namespace', 'search', 'uid'):\n self.__setattr__(meth, self._mk_proxy(meth))\n\n self._update_name()\n self.start()\n\n def _mk_proxy(self, method):\n def proxy_method(*args, **kwargs):\n try:\n safe_assert(self._lock.locked())\n if 'mailbox' in kwargs:\n # We're sharing this connection, so all mailbox methods\n # need to tell us which mailbox they're operating on.\n typ, data = self.select(kwargs['mailbox'])\n if typ.upper() != 'OK':\n return (typ, data)\n del kwargs['mailbox']\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('%s(%s %s)' % (method, args, kwargs))\n rv = getattr(self._conn, method)(*args, **kwargs)\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug((' => %s' % (rv,))[:240])\n return rv\n\n # This is annoyingly repetetive because the imaplib error classes\n # are subclassed in a strange way.\n #\n # In short, we convert imaplib's error, abort and readonly into\n # a subclass of IOError, so Mailplie's common logic can handle\n # things gracefully. In the case of abort, we also kill the\n # connection because it's probably in an unworkable state.\n #\n except IMAP4.readonly:\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n raise IMAP_IOError('Readonly: %s(%s %s)' % (method, args, kwargs))\n except IMAP4.abort:\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n self._shutdown()\n raise IMAP_IOError('Abort: %s(%s %s)' % (method, args, kwargs))\n except IMAP4.error:\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n raise IMAP_IOError('Error: %s(%s %s)' % (method, args, kwargs))\n except:\n # Default is no-op, just re-raise the exception. This includes\n # the assertions above; they're logic errors we don't want to\n # suppress.\n raise\n return proxy_method\n\n def _shutdown(self):\n if self._conn:\n self._conn.shutdown()\n self._conn = None\n self._update_name()\n\n def close(self):\n safe_assert(self._lock.locked())\n self._selected = None\n if '(closed)' not in self.name:\n self.name += ' (closed)'\n return self._conn.close()\n\n def select(self, mailbox='INBOX', readonly=False):\n # This routine caches the SELECT operations, because we will be\n # making lots and lots of superfluous ones \"just in case\" as part\n # of multiplexing one IMAP connection for multiple mailboxes.\n safe_assert(self._lock.locked())\n if self._selected and self._selected[0] == (mailbox, readonly):\n return self._selected[1]\n elif self._selected:\n try:\n self._conn.close()\n except IMAP4.error:\n # This happens if we haven't previously selected a mailbox\n pass\n rv = self._conn.select(mailbox='\"%s\"' % mailbox, readonly=readonly)\n if rv[0].upper() == 'OK':\n info = dict(self._conn.response(f) for f in\n ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'))\n self._selected = ((mailbox, readonly), rv, info)\n else:\n info = '(error)'\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('select(%s, %s) = %s %s'\n % (mailbox, readonly, rv, info))\n return rv\n\n def mailbox_info(self, k, default=None):\n if not self._selected or not self._selected[2]:\n return default\n return self._selected[2].get(k, default)\n\n def _update_name(self):\n name = self._conn and self._conn.host\n if name:\n self.name = name\n elif '(dead)' not in self.name:\n self.name += ' (dead)'\n\n def __enter__(self):\n if not self._conn:\n raise IOError('I am dead')\n self._stop_idling()\n self._lock.acquire()\n self._stop_idling()\n return self\n\n def __exit__(self, type, value, traceback):\n self._start_idling()\n self._lock.release()\n\n def _start_idling(self):\n self._can_idle = True\n\n def _stop_idling(self):\n self._can_idle = False\n\n def _imap_idle(self):\n if not self._conn:\n return\n\n self._can_idle = True\n self.select(self._idle_mailbox)\n if 'imap' in self.session.config.sys.debug:\n logger = self.session.ui.debug\n else:\n logger = lambda x: True\n\n def send_line(data):\n logger('> %s' % data)\n self._conn.send('%s%s' % (data, CRLF))\n\n def get_line():\n data = self._conn._get_line().rstrip()\n logger('< %s' % data)\n return data\n\n try:\n send_line('%s IDLE' % self._conn._new_tag())\n while self._can_idle and not get_line().startswith('+ '):\n pass\n while True:\n rl = wl = xl = None\n try:\n rl, wl, xl = select.select([self._conn.sock], [], [], 1)\n except socket.error:\n pass\n if mailpile.util.QUITTING or not self._can_idle:\n break\n elif rl and self._idle_callback(get_line()):\n self._selected = None\n break\n send_line('DONE')\n # Note: We let the IDLE response drop on the floor, don't care.\n except (socket.error, OSError) as val:\n raise self._conn.abort('socket error: %s' % val)\n\n def quit(self):\n self._can_idle = False # Required to avoid deadlock below\n with self._lock:\n try:\n if self._conn and self._conn.file:\n if self._selected:\n self._conn.close()\n self.logout()\n except (IOError, IMAP4.error, AttributeError):\n pass\n self._can_idle = False\n self._conn = None\n self._update_name()\n\n def run(self):\n try:\n idle_counter = 0\n while self._conn:\n # By default, all this does is send a NOOP every 120 seconds\n # to keep the connection alive (or detect errors).\n for t in range(0, 120):\n time.sleep(1 if self._conn else 0)\n if self._can_idle and self._idle_mailbox:\n idle_counter += 1\n # Once we've been in \"can idle\" state for 5: IDLE!\n if idle_counter >= 5:\n with self as raw_conn:\n self._imap_idle()\n else:\n idle_counter = 0\n\n if self._conn:\n with self as raw_conn:\n raw_conn.noop()\n except:\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n finally:\n self.quit()\n\n\nclass SharedImapMailbox(Mailbox):\n \"\"\"\n This implements a Mailbox view of an IMAP folder. The IMAP connection\n itself is obtained as a SharedImapConn from a particular mail source.\n\n >>> imap = ImapMailSource(session, imap_config)\n >>> mailbox = SharedImapMailbox(session, imap, conn_cls=_MockImap)\n >>> #mailbox.add('From: Bjarni\\\\r\\\\nBarely a message')\n \"\"\"\n\n def __init__(self, session, mail_source,\n mailbox_path='INBOX', conn_cls=None):\n self.config = session\n self.source = mail_source\n self.editable = False # FIXME: this is technically not true\n self.path = mailbox_path\n self.conn_cls = conn_cls\n self._last_updated = None\n self._index = None\n self._factory = None # Unused, for Mailbox compatibility\n self._broken = None\n\n def open_imap(self):\n return self.source.open(throw=IMAP_IOError, conn_cls=self.conn_cls)\n\n def timed_imap(self, *args, **kwargs):\n return self.source.timed_imap(*args, **kwargs)\n\n def last_updated(self):\n return self._last_updated\n\n def _assert(self, test, error):\n if not test:\n raise IMAP_IOError(error)\n\n def __nonzero__(self):\n if self._broken is not None:\n return not self._broken\n try:\n with self.open_imap() as imap:\n ok, data = self.timed_imap(imap.noop, mailbox=self.path)\n self._broken = False\n except (IOError, AttributeError):\n self._broken = True\n return not self._broken\n\n def add(self, message):\n raise Exception('FIXME: Need to RETURN AN ID.')\n self._broken = None\n with self.open_imap() as imap:\n ok, data = self.timed_imap(imap.append, self.path, message=message)\n self._last_updated = time.time()\n self._assert(ok, _('Failed to add message'))\n self._broken = False\n\n def remove(self, key):\n self._broken = None\n with self.open_imap() as imap:\n uidv, uid = (int(k, 36) for k in key.split('.'))\n ok, data = self.timed_imap(imap.uid, 'STORE', uid,\n '+FLAGS', '(\\Deleted)',\n mailbox=self.path)\n self._last_updated = time.time()\n self._assert(ok, _('Failed to remove message'))\n self._broken = False\n\n def mailbox_info(self, k, default=None):\n self._broken = None\n with self.open_imap() as imap:\n imap.select(self.path)\n return imap.mailbox_info(k, default=default)\n self._broken = False\n\n def get_info(self, key):\n self._broken = None\n with self.open_imap() as imap:\n uidv, uid = (int(k, 36) for k in key.split('.'))\n ok, data = self.timed_imap(imap.uid, 'FETCH', uid,\n # Note: It seems that either python's\n # imaplib, or our parser cannot\n # handle dovecot's ENVELOPE\n # details. So omit that for now.\n '(RFC822.SIZE FLAGS)',\n mailbox=self.path)\n if not ok:\n raise KeyError(key)\n self._assert(str(uidv) in imap.mailbox_info('UIDVALIDITY', ['0']),\n _('Mailbox is out of sync'))\n info = dict(zip(*[iter(data[1])]*2))\n info['UIDVALIDITY'] = uidv\n info['UID'] = uid\n self._broken = False\n return info\n\n def get(self, key, _bytes=None):\n info = self.get_info(key)\n if 'UID' not in info:\n raise KeyError(key)\n\n # FIXME: This will hard fail to download mail, if our internet\n # connection averages 8 kbps or worse. Better would be to\n # adapt the chunk size here to actual network performance.\n #\n chunk_size = self.source.timeout * 1024\n chunk = 0\n msg_data = []\n if _bytes and chunk_size > _bytes:\n chunk_size = _bytes\n\n # Some IMAP servers misreport RFC822.SIZE, so we cannot really know\n # how much data to expect. So we just FETCH chunk until one comes up\n # short or empty and assume that's it...\n while chunk >= 0:\n req = '(BODY.PEEK[]<%d.%d>)' % (chunk * chunk_size, chunk_size)\n with self.open_imap() as imap:\n # Note: use the raw method, not the convenient parsed version.\n typ, data = self.source.timed(imap.uid,\n 'FETCH', info['UID'], req,\n mailbox=self.path)\n self._assert(typ == 'OK',\n _('Fetching chunk %d failed') % chunk)\n msg_data.append(data[0][1])\n if len(data[0][1]) < chunk_size:\n chunk = -1\n else:\n chunk += 1\n if _bytes and chunk * chunk_size > _bytes:\n chunk = -1\n\n # FIXME: Should we add a sanity check and complain if we got\n # significantly less data than expected via. RFC822.SIZE?\n return info, ''.join(msg_data)\n\n def get_message(self, key):\n info, payload = self.get(key)\n return Message(payload)\n\n def get_bytes(self, key, *args):\n info, payload = self.get(key, *args)\n return payload\n\n def get_file(self, key):\n info, payload = self.get(key)\n return StringIO.StringIO(payload)\n\n def iterkeys(self):\n self._broken = None\n with self.open_imap() as imap:\n ok, data = self.timed_imap(imap.uid, 'SEARCH', None, 'ALL',\n mailbox=self.path)\n self._assert(ok, _('Failed to list mailbox contents'))\n validity = imap.mailbox_info('UIDVALIDITY', ['0'])[0]\n self._broken = False\n return ('%s.%s' % (b36(int(validity)), b36(int(k)))\n for k in sorted(data))\n\n def keys(self):\n return list(self.iterkeys())\n\n def update_toc(self):\n self._last_updated = time.time()\n\n def get_msg_ptr(self, mboxid, key):\n return '%s%s' % (mboxid, quote(key))\n\n def get_file_by_ptr(self, msg_ptr):\n return self.get_file(unquote(msg_ptr[MBX_ID_LEN:]))\n\n def remove_by_ptr(self, msg_ptr):\n return self.remove(unquote(msg_ptr[MBX_ID_LEN:]))\n\n def get_msg_size(self, key):\n return long(self.get_info(key).get('RFC822.SIZE', 0))\n\n def get_metadata_keywords(self, key):\n # Translate common IMAP flags into the maildir vocabulary\n flags = [f.lower() for f in self.get_info(key).get('FLAGS', '')]\n mkws = []\n for char, flag in (('s', '\\\\seen'),\n ('r', '\\\\answered'),\n ('d', '\\\\draft'),\n ('f', '\\\\flagged'),\n ('t', '\\\\deleted')):\n if flag in flags:\n mkws.append('%s:maildir' % char)\n return mkws\n\n def __contains__(self, key):\n try:\n self.get_info(key)\n return True\n except (KeyError):\n return False\n\n def __len__(self):\n self._broken = None\n with self.open_imap() as imap:\n ok, data = self.timed_imap(imap.noop, mailbox=self.path)\n return imap.mailbox_info('EXISTS', ['0'])[0]\n self._broken = False\n\n def flush(self):\n pass\n\n def close(self):\n pass\n\n def lock(self):\n pass\n\n def unlock(self):\n pass\n\n def save(self, *args, **kwargs):\n # SharedImapMailboxes are never pickled to disk.\n pass\n\n def get_index(self, config, mbx_mid=None):\n if self._index is None:\n self._index = ImapMailboxIndex(config, self, mbx_mid=mbx_mid)\n return self._index\n\n def __unicode__(self):\n if self:\n return _(\"IMAP: %s\") % self.path\n else:\n return _(\"IMAP: %s (not logged in)\") % self.path\n\n def describe_msg_by_ptr(self, msg_ptr):\n if self:\n return _(\"e-mail with ID %s\") % unquote(msg_ptr[MBX_ID_LEN:])\n else:\n return _(\"remote mailbox is inavailable\")\n\n\ndef _connect_imap(session, settings, event,\n conn_cls=None, timeout=30, throw=False,\n logged_in_cb=None, source=None):\n\n def timed(*args, **kwargs):\n if source is not None:\n kwargs['unique_thread'] = 'imap/%s' % (source.my_config._key,)\n return RunTimed(timeout, *args, **kwargs)\n\n def timed_imap(*args, **kwargs):\n if source is not None:\n kwargs['unique_thread'] = 'imap/%s' % (source.my_config._key,)\n return _parse_imap(RunTimed(timeout, *args, **kwargs))\n\n conn = None\n try:\n # Prepare the data section of our event, for keeping state.\n for d in ('mailbox_state',):\n if d not in event.data:\n event.data[d] = {}\n ev = event.data['connection'] = {\n 'live': False,\n 'error': [False, _('Nothing is wrong')]\n }\n\n # If we are given a conn class, use that - this allows mocks for\n # testing.\n if not conn_cls:\n req_stls = (settings.get('protocol') == 'imap_tls')\n want_ssl = (settings.get('protocol') == 'imap_ssl')\n conn_cls = IMAP4_SSL if want_ssl else IMAP4\n else:\n req_stls = want_ssl = False\n\n def mkconn():\n if want_ssl:\n need = [ConnBroker.OUTGOING_IMAPS]\n else:\n need = [ConnBroker.OUTGOING_IMAP]\n with ConnBroker.context(need=need):\n return conn_cls(settings.get('host'),\n int(settings.get('port')))\n conn = timed(mkconn)\n if hasattr(conn, 'sock'):\n conn.sock.settimeout(120)\n conn.debug = ('imaplib' in session.config.sys.debug) and 4 or 0\n\n ok, data = timed_imap(conn.capability)\n if ok:\n capabilities = set(' '.join(data).upper().split())\n else:\n capabilities = set()\n\n if req_stls or ('STARTTLS' in capabilities and not want_ssl):\n try:\n ok, data = timed_imap(conn.starttls)\n if ok:\n # Fetch capabilities again after STARTTLS\n ok, data = timed_imap(conn.capability)\n capabilities = set(' '.join(data).upper().split())\n # Update the protocol to avoid getting downgraded later\n if settings.get('protocol', '') != 'imap_ssl':\n settings['protocol'] = 'imap_tls'\n except (IMAP4.error, IOError, socket.error):\n ok = False\n if not ok:\n ev['error'] = [\n 'tls',\n _('Failed to make a secure TLS connection'),\n '%s:%s' % (settings.get('host'), settings.get('port'))]\n if throw:\n raise throw(ev['error'][1])\n return WithaBool(False)\n\n username = password = \"\"\n try:\n error_type = 'login'\n error_msg = _('IMAP Login error')\n auth_error_type = auth_error_msg = None\n username = settings.get('username', '').encode('utf-8')\n password = IndirectPassword(\n session.config,\n settings.get('password', '')\n ).encode('utf-8')\n\n if (settings.get('auth_type', '').lower() == 'oauth2'\n and 'AUTH=XOAUTH2' in capabilities):\n auth_error_type = 'oauth2'\n auth_error_msg = _('Access denied by mail server')\n token_info = OAuth2.GetFreshTokenInfo(session, username)\n if not (username and token_info and token_info.access_token):\n raise ValueError(\"Missing configuration\")\n ok, data = timed_imap(\n conn.authenticate, 'XOAUTH2',\n lambda challenge: OAuth2.XOAuth2Response(username,\n token_info))\n if not ok:\n token_info.access_token = ''\n\n else:\n auth_error_type = 'auth'\n auth_error_msg = _('Invalid username or password')\n ok, data = timed_imap(conn.login, username, password)\n \n except IMAP4.error as save_error:\n error_str = save_error.__str__()\n ok, data = False, error_str\n if auth_error_type:\n # Is this a password error or some other kind of error?\n # If the response contains any RFC5530 response code\n # check for an RFC5530 auth error code, otherwise check for\n # \"password\" (case independent) in the response string.\n if (re.search('\\[AUTHENTICATIONFAILED\\]|'\n '\\[AUTHORIZATIONFAILED\\]|'\n '\\[EXPIRED\\]', error_str) or\n (not re.search('\\[([a-zA-Z]*)\\]', error_str) and\n re.search('(?i)password', error_str) ) ):\n error_type = auth_error_type\n error_msg = auth_error_msg\n \n except (UnicodeDecodeError, ValueError):\n ok, data = False, None\n if not ok:\n auth_summary = ''\n if source is not None:\n auth_summary = source._summarize_auth()\n ev['error'] = [error_type, error_msg, username, auth_summary]\n if throw:\n raise throw(ev['error'][1])\n return WithaBool(False)\n\n if logged_in_cb is not None:\n logged_in_cb(conn, ev, capabilities)\n\n return conn\n\n except TimedOut:\n if 'imap' in session.config.sys.debug:\n session.ui.debug('%s' % traceback.format_exc())\n ev['error'] = ['timeout', _('Connection timed out')]\n except (ssl.CertificateError, ssl.SSLError):\n if 'imap' in session.config.sys.debug:\n session.ui.debug('%s' % traceback.format_exc())\n ev['error'] = ['tls', _('Failed to make a secure TLS connection'),\n '%s:%s' % (settings.get('host'), settings.get('port'))]\n except (IMAP_IOError, IMAP4.error):\n if 'imap' in session.config.sys.debug:\n session.ui.debug('%s' % traceback.format_exc())\n ev['error'] = ['protocol', _('An IMAP protocol error occurred')]\n except (IOError, AttributeError, socket.error):\n if 'imap' in session.config.sys.debug:\n session.ui.debug('%s' % traceback.format_exc())\n ev['error'] = ['network', _('A network error occurred')]\n\n try:\n if conn:\n # Close the socket directly, in the hopes this will boot\n # any timed-out operations out of a hung state.\n conn.socket().shutdown(socket.SHUT_RDWR)\n conn.file.close()\n except (AttributeError, IOError, socket.error):\n pass\n if throw:\n raise throw(ev['error'])\n\n return None\n\n\nclass ImapMailSource(BaseMailSource):\n \"\"\"\n This is a mail source that connects to an IMAP server.\n\n A single connection is made to the IMAP server, which is then shared\n between the ImapMailSource job and individual mailbox instances.\n \"\"\"\n # This is a helper for the events.\n __classname__ = 'mailpile.mail_source.imap.ImapMailSource'\n\n TIMEOUT_INITIAL = 60\n TIMEOUT_LIVE = 120\n CONN_ERRORS = (IOError, IMAP_IOError, IMAP4.error, TimedOut)\n\n\n class MailSourceVfs(BaseMailSource.MailSourceVfs):\n \"\"\"Expose the IMAP tree to the VFS layer.\"\"\"\n def _imap_path(self, path):\n if path[:1] == '/':\n path = path[1:]\n return path[len(self.root.raw_fp):]\n\n def _imap(self, *args, **kwargs):\n return self.source.timed_imap(*args, **kwargs)\n\n def listdir_(self, where, **kwargs):\n results = []\n path = self._imap_path(where)\n prefix, pathsep = self.source._namespace_info(path)\n with self.source.open() as conn:\n if not conn:\n raise socket.error(_('Not connected to IMAP server.'))\n if path:\n ok, data = self._imap(conn.list, path + pathsep, '%')\n else:\n ok, data = self._imap(conn.list, '', '%')\n while ok and len(data) >= 3:\n (flags, sep, path), data[:3] = data[:3], []\n if path.lower() not in BLACKLISTED_MAILBOXES:\n flags = [f.lower() for f in flags]\n self.source._cache_flags(path, flags)\n results.append('/' + self.source._fmt_path(path))\n return results\n\n def getflags_(self, fp, cfg):\n if self.root == fp:\n return BaseMailSource.MailSourceVfs.getflags_(self, fp, cfg)\n flags = [flag.lower().replace('\\\\', '') for flag in\n self.source._cache_flags(self._imap_path(fp)) or []]\n if not ('hasnochildren' in flags or 'noinferiors' in flags):\n flags.append('Directory')\n if not ('noselect' in flags):\n flags.append('Mailbox')\n return flags\n\n def abspath_(self, fp):\n return fp\n\n def display_name_(self, fp, config):\n return FilePath(fp).display_basename()\n\n def isdir_(self, fp):\n if self.root == fp:\n return True\n flags = self.source._cache_flags(self._imap_path(fp)) or []\n return not ('hasnochildren' in flags or 'noinferiors' in flags)\n\n def getsize_(self, path):\n return None\n\n\n def __init__(self, *args, **kwargs):\n BaseMailSource.__init__(self, *args, **kwargs)\n self.timeout = self.TIMEOUT_INITIAL\n self.last_op = 0\n self.watching = -1\n self.capabilities = set()\n self.logged_in_at = None\n self.namespaces = {'private': []}\n self.flag_cache = {}\n self.conn = None\n self.conn_id = ''\n\n @classmethod\n def Tester(cls, conn_cls, *args, **kwargs):\n tcls = cls(*args, **kwargs)\n return tcls.open(conn_cls=conn_cls) and tcls or False\n\n def timed(self, *args, **kwargs):\n return RunTimed(self.timeout, *args, **kwargs)\n\n def timed_imap(self, *args, **kwargs):\n return _parse_imap(RunTimed(self.timeout, *args, **kwargs))\n\n def _conn_id(self):\n def e(s):\n try:\n return unicode(s).encode('utf-8')\n except UnicodeDecodeError:\n return unicode(s).encode('utf-8', 'replace')\n return md5_hex('\\n'.join([e(self.my_config[k]) for k in\n ('host', 'port', 'password', 'username')]))\n\n def close(self):\n with self._lock:\n if self.conn:\n self.event.data['connection'] = {\n 'live': False,\n 'error': [False, _('Nothing is wrong')]\n }\n self.conn.quit()\n self.conn = None\n\n def open(self, conn_cls=None, throw=False):\n conn = self.conn\n conn_id = self._conn_id()\n if conn:\n try:\n with conn as c:\n now = time.time()\n if (conn_id == self.conn_id and\n (now < self.last_op + 120 or\n self.timed(c.noop)[0] == 'OK')):\n # Make the timeout longer, so we don't drop things\n # on every hiccup and so downloads will be more\n # efficient (chunk size relates to timeout).\n self.timeout = self.TIMEOUT_LIVE\n if now >= self.last_op + 120:\n self.last_op = now\n return conn\n except self.CONN_ERRORS + (AttributeError, ):\n pass\n with self._lock:\n if self.conn == conn:\n self.conn = None\n conn.quit()\n\n my_config = self.my_config\n\n # This facilitates testing, event should already exist in real life.\n if self.event:\n event = self.event\n else:\n event = Event(source=self, flags=Event.RUNNING, data={})\n\n def logged_in_cb(conn, ev, capabilities):\n with self._lock:\n if self.conn is not None:\n raise IOError('Woah, we lost a race.')\n self.capabilities = capabilities\n\n if 'NAMESPACE' in capabilities:\n ok, data = self.timed_imap(conn.namespace)\n if ok:\n prv, oth, shr = data\n self.namespaces = {\n 'private': prv if (prv != 'NIL') else [],\n 'others': oth if (oth != 'NIL') else [],\n 'shared': shr if (shr != 'NIL') else []\n }\n\n if 'IDLE' in capabilities:\n self.conn = SharedImapConn(\n self.session, conn,\n idle_mailbox='INBOX',\n idle_callback=self._idle_callback)\n else:\n self.conn = SharedImapConn(self.session, conn)\n\n if self.event:\n self._log_status(_('Connected to IMAP server %s'\n ) % my_config.host)\n if 'imap' in self.session.config.sys.debug:\n self.session.ui.debug('CONNECTED %s' % self.conn)\n self.session.ui.debug('CAPABILITIES %s' % self.capabilities)\n self.session.ui.debug('NAMESPACES %s' % self.namespaces)\n\n self.conn_id = conn_id\n ev['live'] = True\n\n conn = _connect_imap(self.session, self.my_config, event,\n conn_cls=conn_cls,\n timeout=self.timeout,\n throw=throw,\n logged_in_cb=logged_in_cb,\n source=self)\n if conn:\n self.logged_in_at = time.time()\n return self.conn\n else:\n return WithaBool(False)\n\n def _idle_callback(self, data):\n if 'EXISTS' in data:\n # Stop sleeping and check for mail\n self.wake_up()\n return True\n else:\n return False\n\n def _check_keepalive(self):\n alive_for = time.time() - (self.logged_in_at or time.time())\n if (not self.my_config.keepalive) or alive_for > (12 * 3600):\n if ('IDLE' not in self.capabilities or\n alive_for > self.my_config.interval):\n self.close()\n\n def open_mailbox(self, mbx_id, mfn):\n try:\n proto_me, path = mfn.split('/', 1)\n if proto_me.startswith('src:%s' % self.my_config._key):\n return SharedImapMailbox(self.session, self, mailbox_path=path)\n except ValueError:\n pass\n return None\n\n def _get_mbx_id_and_mfn(self, mbx_cfg):\n mbx_id = FormatMbxId(mbx_cfg._key)\n return mbx_id, self.session.config.sys.mailbox[mbx_id]\n\n def _has_mailbox_changed(self, mbx_cfg, state):\n shared_mbox = self.open_mailbox(*self._get_mbx_id_and_mfn(mbx_cfg))\n uv = state['uv'] = shared_mbox.mailbox_info('UIDVALIDITY', ['0'])[0]\n ex = state['ex'] = shared_mbox.mailbox_info('EXISTS', ['0'])[0]\n uvex = '%s/%s' % (uv, ex)\n if uvex == '0/0':\n return True\n return (uvex != self.event.data.get('mailbox_state',\n {}).get(mbx_cfg._key))\n\n def _mark_mailbox_rescanned(self, mbx, state):\n uvex = '%s/%s' % (state['uv'], state['ex'])\n if 'mailbox_state' in self.event.data:\n self.event.data['mailbox_state'][mbx._key] = uvex\n else:\n self.event.data['mailbox_state'] = {mbx._key: uvex}\n\n def _namespace_info(self, path):\n for which, nslist in self.namespaces.iteritems():\n for prefix, pathsep in nslist:\n if path.startswith(prefix):\n return prefix, pathsep or '/'\n # This is a hack for older servers that don't do NAMESPACE\n if path.startswith('INBOX.'):\n return 'INBOX', '.'\n return '', '/'\n\n def _default_policy(self, mbx_cfg):\n if self._mailbox_path(self._path(mbx_cfg)\n ).lower() in BLACKLISTED_MAILBOXES:\n return 'ignore'\n else:\n return 'inherit'\n\n def _sorted_mailboxes(self):\n # This allows changes to BLACKLISTED_MAILBOXES to have an effect\n # even if peoples' configs say otherwise.\n return [\n m for m in BaseMailSource._sorted_mailboxes(self)\n if m.name.lower() not in BLACKLISTED_MAILBOXES]\n\n def _msg_key_order(self, key):\n return [int(k, 36) for k in key.split('.')]\n\n def _strip_file_extension(self, mbx_path):\n return mbx_path # Yes, a no-op :)\n\n def _decode_path(self, path):\n try:\n return path.decode('imap4-utf-7')\n except:\n return path\n\n def _mailbox_path(self, mbx_path):\n # len('src:/') = 5\n return str(mbx_path[(5 + len(self.my_config._key)):])\n\n def _mailbox_path_split(self, mbx_path):\n path = self._mailbox_path(mbx_path)\n prefix, pathsep = self._namespace_info(path)\n return [self._decode_path(p) for p in path.split(pathsep)]\n\n def _mailbox_name(self, mbx_path):\n path = self._mailbox_path(mbx_path)\n prefix, pathsep = self._namespace_info(path)\n return self._decode_path(path[len(prefix):])\n\n def _fmt_path(self, path):\n return 'src:%s/%s' % (self.my_config._key, path)\n\n def _fix_empty_discovery_path_bug(self):\n if self.my_config.discovery.policy not in ('unknown', 'ignore'):\n if not self.my_config.discovery.paths:\n self.my_config.discovery.paths.append('/')\n\n def discover_mailboxes(self, paths=None):\n config = self.session.config\n ostate = self.on_event_discovery_starting()\n self._fix_empty_discovery_path_bug()\n try:\n paths = copy.copy(paths or self.my_config.discovery.paths)\n max_mailboxes = self.my_config.discovery.max_mailboxes\n mailbox_count = len(self.my_config.mailbox)\n existing = self._existing_mailboxes()\n mailboxes = []\n\n with self.open() as raw_conn:\n for p in paths:\n mailboxes += self._walk_mailbox_path(raw_conn, str(p))\n\n discovered = [mbx for mbx in mailboxes if mbx not in existing]\n if discovered and (len(discovered) > max_mailboxes - mailbox_count):\n discovered = discovered[:max(0, max_mailboxes - mailbox_count)]\n if self.on_event_discovery_toomany():\n return self.discover_mailboxes(paths=paths)\n\n self.set_event_discovery_state('adding')\n for path in discovered:\n idx = config.sys.mailbox.append(path)\n mbx = self.take_over_mailbox(idx)\n\n return len(discovered)\n except:\n if config.sys.debug:\n self.session.ui.debug('%s' % traceback.format_exc())\n raise\n finally:\n self.on_event_discovery_done(ostate)\n\n def _cache_flags(self, path, flags=None):\n path = self._fmt_path(path)\n if flags is not None:\n self.flag_cache[path] = flags\n return self.flag_cache.get(path)\n\n def _walk_mailbox_path(self, conn, prefix):\n \"\"\"\n Walks the IMAP path recursively and returns a list of all found\n mailboxes.\n \"\"\"\n mboxes = []\n subtrees = []\n # We go well over the maximum here, so the calling code can detect\n # detect that we want to go over the limits and can ask the user\n # whether that's OK.\n max_mailboxes = 5 + (2 * self.my_config.discovery.max_mailboxes)\n if prefix == '/':\n prefix = ''\n try:\n ok, data = self.timed_imap(conn.list, prefix, '%')\n while ok and len(data) >= 3:\n (flags, sep, path), data[:3] = data[:3], []\n flags = [f.lower() for f in flags]\n if '\\\\noselect' not in flags:\n if path.lower() not in BLACKLISTED_MAILBOXES:\n # We cache the flags for this mailbox, they may tell\n # use useful things about what kind of mailbox it is.\n self._cache_flags(path, flags)\n mboxes.append(self._fmt_path(path))\n if '\\\\haschildren' in flags:\n subtrees.append('%s%s' % (path, sep))\n if len(mboxes) > max_mailboxes:\n break\n for path in subtrees:\n if len(mboxes) <= max_mailboxes:\n mboxes.extend(self._walk_mailbox_path(conn, path))\n except self.CONN_ERRORS:\n pass\n finally:\n return mboxes\n\n def quit(self, *args, **kwargs):\n if self.conn:\n self.conn.quit()\n return BaseMailSource.quit(self, *args, **kwargs)\n\n\ndef TestImapSettings(session, settings, event,\n timeout=ImapMailSource.TIMEOUT_INITIAL):\n conn = _connect_imap(session, settings, event, timeout=timeout)\n if conn:\n try:\n conn.socket().shutdown(socket.SHUT_RDWR)\n conn.file.close()\n except (IOError, OSError, socket.error):\n pass\n return True\n return False\n\n\n##[ Test code follows ]#######################################################\n\nclass _MockImap(object):\n \"\"\"\n Base mock that pretends to be an imaplib IMAP connection.\n\n >>> imap = ImapMailSource(session, imap_config)\n >>> imap.open(conn_cls=_MockImap)\n \n\n >>> sorted(imap.capabilities)\n ['IMAP4REV1', 'X-MAGIC-BEANS']\n \"\"\"\n DEFAULT_RESULTS = {\n 'append': ('OK', []),\n 'capability': ('OK', ['X-MAGIC-BEANS', 'IMAP4rev1']),\n 'list': ('OK', []),\n 'login': ('OK', ['\"Welcome, human\"']),\n 'noop': ('OK', []),\n 'select': ('OK', []),\n }\n RESULTS = {}\n\n def __init__(self, *args, **kwargs):\n self.host = 'mock'\n def mkcmd(rval):\n def cmd(*args, **kwargs):\n return rval\n return cmd\n for cmd, rval in dict_merge(self.DEFAULT_RESULTS, self.RESULTS\n ).iteritems():\n self.__setattr__(cmd, mkcmd(rval))\n\n def __getattr__(self, attr):\n return self.__getattribute__(attr)\n\n\nclass _Mocks(object):\n \"\"\"\n A bunch of IMAP test classes for testing various configurations.\n\n >>> ImapMailSource.Tester(_Mocks.NoDns, session, imap_config)\n False\n\n >>> ImapMailSource.Tester(_Mocks.BadLogin, session, imap_config)\n False\n \"\"\"\n class NoDns(_MockImap):\n def __init__(self, *args, **kwargs):\n raise socket.gaierror('Oops')\n\n class BadLogin(_MockImap):\n RESULTS = {'login': ('BAD', ['\"Sorry dude\"'])}\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n import mailpile.config.defaults\n import mailpile.config.manager\n import mailpile.ui\n\n rules = mailpile.config.defaults.CONFIG_RULES\n config = mailpile.config.manager.ConfigManager(rules=rules)\n config.sources.imap = {\n 'protocol': 'imap_ssl',\n 'host': 'imap.gmail.com',\n 'port': 993,\n 'username': 'nobody',\n 'password': 'nowhere'\n }\n session = mailpile.ui.Session(config)\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={'session': session,\n 'imap_config': config.sources.imap})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n\n args = sys.argv[1:]\n if args:\n session.config.sys.debug = 'imap'\n\n username, password = args.pop(0), args.pop(0)\n config.sources.imap.username = username\n config.sources.imap.password = password\n imap = ImapMailSource(session, config.sources.imap)\n with imap.open(throw=IMAP_IOError) as conn:\n print('%s' % (conn.list(), ))\n mbx = SharedImapMailbox(config, imap, mailbox_path='INBOX')\n print('%s' % list(mbx.iterkeys()))\n for key in args:\n info, payload = mbx.get(key)\n print('%s(%d bytes) = %s\\n%s' % (mbx.get_msg_ptr('0000', key),\n mbx.get_msg_size(key),\n info, payload))\n" }, { "path": "mailpile/mail_source/imap_starttls.py", "content": "import imaplib\nimport ssl\n\nCommands = {\n 'STARTTLS': ('NONAUTH')\n}\n\nimaplib.Commands.update(Commands)\n\nclass IMAP4(imaplib.IMAP4, object):\n\n # This is a bugfix for imaplib's default readline method. It is\n # identical except it raises abort() instead of error() as the\n # internal state will certainly be broken.\n #\n # Note: This would be the function to change if we want to do away\n # with the line-length limits altogether.\n #\n def readline(self):\n \"\"\"Read line from remote.\"\"\"\n line = self.file.readline(imaplib._MAXLINE + 1)\n if len(line) > imaplib._MAXLINE:\n raise self.abort(\"got more than %d bytes\" % imaplib._MAXLINE)\n return line\n\n def starttls(self, keyfile = None, certfile = None):\n typ, data = self._simple_command('STARTTLS')\n if typ != 'OK':\n raise self.error('no STARTTLS')\n self.sock = ssl.wrap_socket(self.sock, keyfile, certfile)\n self.file = self.sock.makefile('rb')\n return typ, data\n" }, { "path": "mailpile/mail_source/imap_utf7.py", "content": "# -*- coding: utf-8- -*-\n\n# from: http://piao-tech.blogspot.no/2010/03/get-offlineimap-working-with-non-ascii.html#resources\n\nimport binascii\nimport codecs\n\n# encoding\n\ndef modified_base64 (s):\n s = s.encode('utf-16be')\n return binascii.b2a_base64(s).rstrip('\\n=').replace('/', ',')\n\ndef doB64(_in, r):\n if _in:\n r.append('&%s-' % modified_base64(''.join(_in)))\n del _in[:]\n\ndef encoder(s):\n r = []\n _in = []\n for c in s:\n ordC = ord(c)\n if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e:\n doB64(_in, r)\n r.append (c)\n elif c == '&':\n doB64(_in, r)\n r.append ('&-')\n else:\n _in.append(c)\n doB64(_in, r)\n return (str(''.join(r)), len(s))\n\n# decoding\ndef modified_unbase64(s):\n b = binascii.a2b_base64(s.replace(',', '/') + '===')\n return unicode (b, 'utf-16be')\n\ndef decoder (s):\n r = []\n decode = []\n for c in s:\n if c == '&' and not decode:\n decode.append ('&')\n elif c == '-' and decode:\n if len(decode) == 1:\n r.append('&')\n else:\n r.append(modified_unbase64(''.join(decode[1:])))\n decode = []\n elif decode:\n decode.append(c)\n else:\n r.append(c)\n\n if decode:\n r.append(modified_unbase64(''.join(decode[1:])))\n bin_str = ''.join(r)\n return (bin_str, len(s))\n\nclass StreamReader (codecs.StreamReader):\n def decode (self, s, errors='strict'):\n return decoder(s)\n\nclass StreamWriter (codecs.StreamWriter):\n def decode (self, s, errors='strict'):\n return encoder(s)\n\ndef imap4_utf_7(name):\n if name == 'imap4-utf-7':\n return (encoder, decoder, StreamReader, StreamWriter)\n\ncodecs.register(imap4_utf_7)\n" }, { "path": "mailpile/mail_source/local.py", "content": "import time\nimport os\n\nfrom mailpile.mail_source import BaseMailSource\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.vfs import FilePath\n\n\nclass LocalMailSource(BaseMailSource):\n \"\"\"\n This is a mail source that watches over one or more local mailboxes.\n \"\"\"\n # This is a helper for the events.\n __classname__ = 'mailpile.mail_source.local.LocalMailSource'\n\n def __init__(self, *args, **kwargs):\n BaseMailSource.__init__(self, *args, **kwargs)\n if not self.my_config.name:\n self.my_config.name = _('Local mail')\n self.recently_changed = []\n self.my_config.protocol = 'local' # We may be upgrading an old\n # mbox or maildir source.\n self.watching = -1\n\n def _sleeping_is_ok(self, slept):\n if slept > 5:\n #\n # If any of the most recently changed mailboxes has changed\n # again, cut our sleeps short after 5 seconds. By basing this\n # on recently changed mailboxes, we don't need to explicitly\n # ask the user which mailbox(es) are being used as Inboxes.\n #\n # The number 10 should be \"big enough\", without us going\n # nuts and scanning a gazillion mailboxes every second.\n #\n if len(self.recently_changed) > 10:\n self.recently_changed = self.recently_changed[-10:]\n for mbx in self.recently_changed:\n if self._has_mailbox_changed(mbx, {}):\n return False\n return True\n\n def close(self):\n pass\n\n def open(self):\n with self._lock:\n mailboxes = self.my_config.mailbox.values()\n if self.watching == len(mailboxes):\n return True\n else:\n self.watching = len(mailboxes)\n\n # Prepare the data section of our event, for keeping state.\n for d in ('mailbox_state', ):\n if d not in self.event.data:\n self.event.data[d] = {}\n\n self._log_status(_('Watching %d mbox mailboxes') % self.watching)\n return True\n\n def _get_macmaildir_data(self, path):\n ds = [d for d in os.listdir(path) if not d.startswith('.')\n and os.path.isdir(os.path.join(path, d))]\n return (len(ds) == 1) and os.path.join(path, ds[0], 'Data')\n\n def _data_paths(self, mbx):\n mbx_path = FilePath(self._path(mbx)).raw_fp\n if os.path.exists(mbx_path):\n yield mbx_path\n\n if os.path.isdir(mbx_path):\n # Maildir, WERVD\n for s in ('cur', 'new', 'tmp', 'wervd.ver'):\n sub_path = os.path.join(mbx_path, s)\n if os.path.exists(sub_path):\n yield sub_path\n\n # Mac Maildir\n sub_path = self._get_macmaildir_data(mbx_path)\n if sub_path:\n yield sub_path\n\n def _mailbox_sort_key(self, mbx):\n # Sort mailboxes so the most recently modified get scanned first.\n mt = 0\n for p in self._data_paths(mbx):\n try:\n mt = max(mt, os.path.getmtime(p))\n except (OSError, IOError):\n pass\n if mt:\n return '%20.20d' % (0x10000000000 - long(mt))\n else:\n return BaseMailSource._mailbox_sort_key(self, mbx)\n\n def _has_mailbox_changed(self, mbx, state):\n mtszs = []\n for p in self._data_paths(mbx):\n try:\n mt = long(os.path.getmtime(p))\n sz = long(os.path.getsize(p))\n mtszs.append('%s/%s' % (mt, sz))\n except (OSError, IOError):\n pass\n\n if not mtszs:\n # Try to rescan even if the above fails for some reason\n mt = sz = (int(time.time()) // 7200)\n mtszs = ['%s/%s' % (mt, sz)]\n\n mtsz = state['mtsz'] = ','.join(mtszs)\n if (mtsz != self.event.data.get('mailbox_state', {}).get(mbx._key)):\n while mbx in self.recently_changed:\n self.recently_changed.remove(mbx)\n self.recently_changed.append(mbx)\n return True\n else:\n return False\n\n def _mark_mailbox_rescanned(self, mbx, state):\n if 'mailbox_state' in self.event.data:\n self.event.data['mailbox_state'][mbx._key] = state['mtsz']\n else:\n self.event.data['mailbox_state'] = {mbx._key: state['mtsz']}\n\n def _is_mbox(self, fn):\n try:\n with open(fn, 'rb') as fd:\n data = fd.read(2048) # No point reading less...\n if data.startswith('From '):\n # OK, this might be an mbox! Let's check if the first\n # few lines look like RFC2822 headers...\n headcount = 0\n for line in data.splitlines(True)[1:]:\n if (headcount > 2) and line in ('\\n', '\\r\\n'):\n return True\n if line[-1:] == '\\n' and line[:1] not in (' ', '\\t'):\n parts = line.split(':')\n if (len(parts) < 2 or\n ' ' in parts[0] or '\\t' in parts[0]):\n return False\n headcount += 1\n return (headcount > 2)\n except (IOError, OSError):\n pass\n return False\n\n def _is_maildir(self, fn):\n if not os.path.isdir(fn):\n return False\n for sub in ('cur', 'new', 'tmp'):\n subdir = os.path.join(fn, sub)\n if not os.path.exists(subdir) or not os.path.isdir(subdir):\n return False\n return True\n\n def _is_macmaildir(self, path):\n infoplist = os.path.join(path, 'Info.plist')\n if not os.path.isdir(path) or not os.path.exists(infoplist):\n return False\n data = self._get_macmaildir_data(path)\n return data and os.path.isdir(data)\n\n def is_mailbox(self, fn):\n fn = FilePath(fn).raw_fp\n return (self._is_maildir(fn) or\n self._is_macmaildir(fn) or\n self._is_mbox(fn))\n" }, { "path": "mailpile/mail_source/pop3.py", "content": "import os\nimport ssl\nimport traceback\n\nfrom mailpile.auth import IndirectPassword\nfrom mailpile.conn_brokers import Master as ConnBroker\nfrom mailpile.mail_source import BaseMailSource\nfrom mailpile.mailboxes import pop3\nfrom mailpile.mailutils import FormatMbxId, MBX_ID_LEN\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.util import *\n\n\n# We use this to enable \"recent mode\" on GMail accounts by default.\nGMAIL_TLDS = ('gmail.com', 'googlemail.com')\n\n\ndef _open_pop3_mailbox(session, event, host, port,\n username, password, auth_type,\n protocol, debug, throw=False):\n cev = event.data['connection'] = {\n 'live': False,\n 'error': [False, _('Nothing is wrong')]\n }\n try:\n # FIXME: Nothing actually adds gmail or gmail-full to the protocol\n # yet, so we're stuck in recent mode only for now.\n if (username.lower().split('@')[-1] in GMAIL_TLDS\n or 'gmail' in protocol):\n if 'gmail-full' not in protocol:\n username = 'recent:%s' % username\n\n if 'ssl' in protocol:\n need = [ConnBroker.OUTGOING_POP3S]\n else:\n need = [ConnBroker.OUTGOING_POP3]\n\n with ConnBroker.context(need=need):\n return pop3.MailpileMailbox(host,\n port=port,\n user=username,\n password=password,\n auth_type=auth_type,\n use_ssl=('ssl' in protocol),\n session=session,\n debug=debug)\n except AccessError:\n cev['error'] = ['auth', _('Invalid username or password'),\n username, sha1b64(password)]\n except (ssl.CertificateError, ssl.SSLError):\n cev['error'] = ['tls', _('Failed to make a secure TLS connection'),\n '%s:%s' % (host, port)]\n except (IOError, OSError):\n cev['error'] = ['network', _('A network error occurred')]\n event.data['traceback'] = traceback.format_exc()\n\n if throw:\n raise throw(cev['error'][1])\n\n return None\n\n\nclass POP3_IOError(IOError):\n pass\n\n\nclass Pop3MailSource(BaseMailSource):\n \"\"\"\n This is a mail source that watches over one or more POP3 mailboxes.\n \"\"\"\n # This is a helper for the events.\n __classname__ = 'mailpile.mail_source.pop3.Pop3MailSource'\n\n def __init__(self, *args, **kwargs):\n BaseMailSource.__init__(self, *args, **kwargs)\n self.watching = -1\n\n def close(self):\n mbx = self.my_config.mailbox.values()[0]\n if mbx:\n pop3 = self.session.config.open_mailbox(self.session,\n FormatMbxId(mbx._key),\n prefer_local=False,\n from_cache=True)\n if pop3:\n pop3.close() \n\n def _sleep(self, *args, **kwargs):\n self.close()\n return BaseMailSource._sleep(self, *args, **kwargs)\n\n def open(self):\n with self._lock:\n mailboxes = self.my_config.mailbox.values()\n if self.watching == len(mailboxes):\n return True\n else:\n self.watching = len(mailboxes)\n\n for d in ('mailbox_state', ):\n if d not in self.event.data:\n self.event.data[d] = {}\n self.event.data['connection'] = {\n 'live': False,\n 'error': [False, _('Nothing is wrong')]\n }\n\n self._log_status(_('Watching %d POP3 mailboxes') % self.watching)\n return True\n\n def _has_mailbox_changed(self, mbx, state):\n pop3 = self.session.config.open_mailbox(self.session,\n FormatMbxId(mbx._key),\n prefer_local=False)\n state['stat'] = stat = '%s' % (pop3.stat(), )\n return (self.event.data.get('mailbox_state', {}).get(mbx._key) != stat)\n\n def _mark_mailbox_rescanned(self, mbx, state):\n if 'mailbox_state' in self.event.data:\n self.event.data['mailbox_state'][mbx._key] = state['stat']\n else:\n self.event.data['mailbox_state'] = {mbx._key: state['stat']}\n\n def _fmt_path(self):\n return 'src:%s' % (self.my_config._key,)\n\n def open_mailbox(self, mbx_id, mfn):\n my_cfg = self.my_config\n if 'src:' in mfn[:5] and FormatMbxId(mbx_id) in my_cfg.mailbox:\n debug = ('pop3' in self.session.config.sys.debug) and 99 or 0\n password = IndirectPassword(self.session.config, my_cfg.password)\n return _open_pop3_mailbox(self.session, self.event,\n my_cfg.host, my_cfg.port,\n my_cfg.username, password,\n my_cfg.auth_type,\n my_cfg.protocol, debug,\n throw=POP3_IOError)\n return None\n\n def discover_mailboxes(self, paths=None):\n config = self.session.config\n existing = self._existing_mailboxes()\n if self._fmt_path() not in existing:\n idx = config.sys.mailbox.append(self._fmt_path())\n self.take_over_mailbox(idx)\n return 1\n return 0\n\n def is_mailbox(self, fn):\n return False\n\n def _mailbox_name(self, path):\n return _(\"Inbox\")\n\n def _create_tag(self, *args, **kwargs):\n ptag = kwargs.get('parent')\n try:\n if ptag:\n return self.session.config.get_tags(ptag)[0]._key\n except (IndexError, KeyError):\n pass\n return BaseMailSource._create_tag(self, *args, **kwargs)\n\n\ndef TestPop3Settings(session, settings, event):\n password = IndirectPassword(session.config, settings['password'])\n conn = _open_pop3_mailbox(session, event,\n settings['host'],\n int(settings['port']),\n settings['username'],\n password,\n settings.get('auth_type', 'password'),\n settings['protocol'],\n True)\n if conn:\n conn.close()\n return True\n return False\n" }, { "path": "mailpile/mailboxes/__init__.py", "content": "## Dear hackers!\n##\n## It would be great to have more mailbox classes. They should be derived\n## from or implement the same interfaces as Python's native mailboxes, with\n## the additional constraint that they support pickling and unpickling using\n## cPickle. The mailbox class is also responsible for generating and parsing\n## a \"pointer\" which should be a short as possible while still encoding the\n## info required to locate this message and this message only within the\n## larger mailbox.\n\nimport time\nfrom urllib import quote, unquote\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.index.mailboxes import MailboxIndex\nfrom mailpile.mailutils import MBX_ID_LEN\nfrom mailpile.util import MboxRLock\n\n\n__all__ = ['mbox', 'maildir', 'gmvault', 'macmail', 'pop3', 'wervd',\n 'MBX_ID_LEN',\n 'NoSuchMailboxError', 'IsMailbox', 'OpenMailbox']\n\nMAILBOX_CLASSES = []\n\n\nclass NoSuchMailboxError(OSError):\n pass\n\n\ndef register(prio, cls):\n global MAILBOX_CLASSES\n MAILBOX_CLASSES.append((prio, cls))\n MAILBOX_CLASSES.sort()\n\n\ndef IsMailbox(fn, config):\n for pri, mbox_cls in MAILBOX_CLASSES:\n try:\n if mbox_cls.parse_path(config, fn):\n return (True, mbox_cls)\n except KeyboardInterrupt:\n raise\n except:\n pass\n return False\n\n\ndef OpenMailbox(fn, config, create=False):\n for pri, mbox_cls in MAILBOX_CLASSES:\n try:\n return mbox_cls(\n *mbox_cls.parse_path(config, fn, create=create, allow_empty=True))\n except KeyboardInterrupt:\n raise\n except:\n pass\n raise ValueError('Not a mailbox: %s' % fn)\n\n\ndef UnorderedPicklable(parent, editable=False):\n \"\"\"A factory for generating unordered, picklable mailbox classes.\"\"\"\n\n class UnorderedPicklableMailbox(parent):\n UNPICKLABLE = []\n\n def __init__(self, *args, **kwargs):\n parent.__init__(self, *args, **kwargs)\n self.editable = editable\n self.source_map = {}\n self.is_local = False\n self._last_updated = None\n self._lock = MboxRLock()\n self._index = None\n self._save_to = None\n self._encryption_key_func = lambda: None\n self._decryption_key_func = lambda: None\n self.__init2__(*args, **kwargs)\n\n\n def __init2__(self, *args, **kwargs):\n pass\n\n def __enter__(self, *args, **kwargs):\n self._lock.acquire()\n return self\n\n def __exit__(self, *args, **kwargs):\n self._lock.release()\n\n def __unicode__(self):\n return unicode(str(self))\n\n def describe_msg_by_ptr(self, msg_ptr):\n try:\n return self._describe_msg_by_ptr(msg_ptr)\n except KeyError:\n return _(\"message not found in mailbox\")\n\n def _describe_msg_by_ptr(self, msg_ptr):\n return unicode(msg_ptr)\n\n def __setstate__(self, data):\n self.__dict__.update(data)\n self._lock = MboxRLock()\n with self._lock:\n self._index = None\n self._save_to = None\n self._encryption_key_func = lambda: None\n self._decryption_key_func = lambda: None\n if not hasattr(self, 'source_map'):\n self.source_map = {}\n if (len(self.source_map) > 0 and\n not hasattr(self, 'is_local') or not self.is_local):\n self.is_local = True\n self.update_toc()\n\n def __getstate__(self):\n odict = self.__dict__.copy()\n # Pickle can't handle function objects.\n for dk in ['_save_to', '_index', '_last_updated',\n '_encryption_key_func', '_decryption_key_func',\n '_file', '_lock', 'parsed'] + self.UNPICKLABLE:\n if dk in odict:\n del odict[dk]\n return odict\n\n def save(self, session=None, to=None, pickler=None):\n with self._lock:\n if to and pickler:\n self._save_to = (pickler, to)\n if self._save_to and len(self) > 0:\n pickler, fn = self._save_to\n if session:\n session.ui.mark(_('Saving %s state to %s')\n % (self, fn))\n pickler(self, fn)\n\n def add_from_source(self, source_id, metadata_kws, *args, **kwargs):\n with self._lock:\n key = self.add(*args, **kwargs)\n self.set_metadata_keywords(key, metadata_kws)\n self.source_map[source_id] = key\n return key\n\n def update_toc(self):\n self._last_updated = time.time()\n self._refresh()\n self._last_updated = time.time()\n\n def last_updated(self):\n return self._last_updated\n\n def get_msg_ptr(self, mboxid, toc_id):\n return '%s%s' % (mboxid, quote(toc_id))\n\n def get_file(self, *args, **kwargs):\n with self._lock:\n return parent.get_file(self, *args, **kwargs)\n\n def get_file_by_ptr(self, msg_ptr):\n return self.get_file(unquote(msg_ptr[MBX_ID_LEN:]))\n\n def remove_by_ptr(self, msg_ptr):\n self._last_updated = time.time()\n return self.remove(unquote(msg_ptr[MBX_ID_LEN:]))\n\n def get_msg_size(self, toc_id):\n with self._lock:\n fd = self.get_file(toc_id)\n fd.seek(0, 2)\n return fd.tell()\n\n def get_bytes(self, toc_id, *args):\n with self._lock:\n return self.get_file(toc_id).read(*args)\n\n def get_string(self, *args, **kwargs):\n with self._lock:\n return parent.get_string(self, *args, **kwargs)\n\n def get_metadata_keywords(self, toc_id):\n # Subclasses should translate whatever internal metadata they\n # have into mailpile keywords describing message metadata\n return []\n\n def set_metadata_keywords(self, toc_id, kws):\n pass\n\n def get_index(self, config, mbx_mid=None):\n with self._lock:\n if self._index is None:\n self._index = MailboxIndex(config, self, mbx_mid=mbx_mid)\n return self._index\n\n def remove(self, *args, **kwargs):\n with self._lock:\n self._last_updated = time.time()\n return parent.remove(self, *args, **kwargs)\n\n def _get_fd(self, *args, **kwargs):\n with self._lock:\n return parent._get_fd(self, *args, **kwargs)\n\n def _refresh(self, *args, **kwargs):\n with self._lock:\n return parent._refresh(self, *args, **kwargs)\n\n def __setitem__(self, *args, **kwargs):\n with self._lock:\n self._last_updated = time.time()\n return parent.__setitem__(self, *args, **kwargs)\n\n def __getitem__(self, *args, **kwargs):\n with self._lock:\n return parent.__getitem__(self, *args, **kwargs)\n\n\n return UnorderedPicklableMailbox\n" }, { "path": "mailpile/mailboxes/gmvault.py", "content": "import mailbox\nimport os\nimport gzip\nimport rfc822\n\nimport mailpile.mailboxes\nimport mailpile.mailboxes.maildir as maildir\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nclass MailpileMailbox(maildir.MailpileMailbox):\n \"\"\"A Gmvault class that supports pickling and a few mailpile specifics.\"\"\"\n\n @classmethod\n def parse_path(cls, config, fn, create=False, allow_empty=False):\n if (os.path.isdir(fn) and\n os.path.isdirs(os.path.join(fn, 'db')) and\n os.path.isdirs(os.path.join(fn, 'chats')) and\n os.path.isdirs(os.path.join(fn, '.info'))):\n return (fn, )\n raise ValueError('Not a Gmvault: %s' % fn)\n\n def __init__(self, dirname, factory=rfc822.Message, create=True):\n maildir.MailpileMailbox.__init__(self, dirname, factory, create)\n self._paths = {'db': os.path.join(self._path, 'db')}\n self._toc_mtimes = {'db': 0}\n\n def get_file(self, key):\n \"\"\"Return a file-like representation or raise a KeyError.\"\"\"\n fname = self._lookup(key)\n if fname.endswith('.gz'):\n f = gzip.open(os.path.join(self._path, fname), 'rb')\n else:\n f = open(os.path.join(self._path, fname), 'rb')\n return mailbox._ProxyFile(f)\n\n def _refresh(self):\n \"\"\"Update table of contents mapping.\"\"\"\n # Refresh toc\n self._toc = {}\n for path in self._paths:\n for dirpath, dirnames, filenames in os.walk(self._paths[path]):\n for filename in [f for f in filenames\n if f.endswith(\".eml.gz\")\n or f.endswith(\".eml\")]:\n self._toc[filename] = os.path.join(dirpath, filename)\n\n\nmailpile.mailboxes.register(50, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/macmail.py", "content": "import mailbox\nimport sys\nimport os\nimport warnings\nimport rfc822\nimport time\nimport errno\n\nimport mailpile.mailboxes\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import UnorderedPicklable\n\n\nclass _MacMaildirPartialFile(mailbox._PartialFile):\n def __init__(self, fd):\n length = int(fd.readline().strip())\n start = fd.tell()\n stop = start+length\n mailbox._PartialFile.__init__(self, fd, start=start, stop=stop)\n\n\nclass MacMaildirMessage(mailbox.Message):\n def __init__(self, message=None):\n if hasattr(message, \"read\"):\n length = int(message.readline().strip())\n message = message.read(length)\n\n mailbox.Message.__init__(self, message)\n\n\nclass MacMaildir(mailbox.Mailbox):\n def __init__(self, dirname, factory=rfc822.Message, create=True):\n mailbox.Mailbox.__init__(self, dirname, factory, create)\n if not os.path.exists(self._path):\n if create:\n raise NotImplemented(\"Why would we support creation of \"\n \"silly mailboxes?\")\n else:\n raise mailbox.NoSuchMailboxError(self._path)\n\n # What have we here?\n ds = os.listdir(self._path)\n\n # Okay, MacMaildirs have Info.plist files\n if not 'Info.plist' in ds:\n raise mailbox.FormatError(self._path)\n\n # Now ignore all the files and dotfiles...\n ds = [d for d in ds if not d.startswith('.')\n and os.path.isdir(os.path.join(self._path, d))]\n\n # There should be exactly one directory left, which is our \"ID\".\n if len(ds) == 1:\n self._id = ds[0]\n else:\n raise mailbox.FormatError(self._path)\n\n # And finally, there's a Data folder (with .emlx files in it)\n self._mailroot = \"%s/%s/Data/\" % (self._path, self._id)\n if not os.path.isdir(self._mailroot):\n raise mailbox.FormatError(self._path)\n\n self._toc = {}\n self._last_read = 0\n\n def remove(self, key):\n \"\"\"Remove the message or raise error if nonexistent.\"\"\"\n safe_remove(os.path.join(self._mailroot, self._lookup(key)))\n try:\n del self._toc[key]\n except:\n pass\n\n def discard(self, key):\n \"\"\"If the message exists, remove it.\"\"\"\n try:\n self.remove(key)\n except KeyError:\n pass\n except OSError as e:\n if e.errno != errno.ENOENT:\n raise\n\n def __setitem__(self, key, message):\n \"\"\"Replace a message\"\"\"\n raise NotImplemented(\"Mailpile is readonly, for now.\")\n\n def iterkeys(self):\n self._refresh()\n for key in self._toc:\n try:\n self._lookup(key)\n except KeyError:\n continue\n yield key\n\n def has_key(self, key):\n self._refresh()\n return key in self._toc\n\n def __len__(self):\n self._refresh()\n return len(self._toc)\n\n def _refresh(self):\n self._toc = {}\n paths = [\"\"]\n\n while not paths == []:\n curpath = paths.pop(0)\n fullpath = os.path.join(self._mailroot, curpath)\n try:\n for entry in os.listdir(fullpath):\n p = os.path.join(fullpath, entry)\n if os.path.isdir(p):\n paths.append(os.path.join(curpath, entry))\n elif entry[-5:] == \".emlx\":\n self._toc[entry[:-5]] = os.path.join(curpath, entry)\n except (OSError, IOError):\n pass # Ignore difficulties reading individual folders\n\n def _lookup(self, key):\n try:\n if os.path.exists(os.path.join(self._mailroot, self._toc[key])):\n return self._toc[key]\n except KeyError:\n pass\n self._refresh()\n try:\n return self._toc[key]\n except KeyError:\n raise KeyError(\"No message with key %s\" % key)\n\n def get_message(self, key):\n f = open(os.path.join(self._mailroot, self._lookup(key)), 'r')\n msg = MacMaildirMessage(f)\n f.close()\n return msg\n\n def get_file(self, key):\n f = open(os.path.join(self._mailroot, self._lookup(key)), 'r')\n return _MacMaildirPartialFile(f)\n\n\nclass MailpileMailbox(UnorderedPicklable(MacMaildir)):\n \"\"\"A Mac Mail.app maildir class that supports pickling etc.\"\"\"\n @classmethod\n def parse_path(cls, config, fn, create=False, allow_empty=False):\n if (os.path.isdir(fn)\n and os.path.exists(os.path.join(fn, 'Info.plist'))):\n return (fn, )\n raise ValueError('Not a Mac Mail.app Maildir: %s' % fn)\n\n def __unicode__(self):\n return _(\"Mac Maildir %s\") % self._mailroot\n\n def _describe_msg_by_ptr(self, msg_ptr):\n return _(\"e-mail in file %s\") % self._lookup(msg_ptr[MBX_ID_LEN:])\n\n\nmailpile.mailboxes.register(50, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/maildir.py", "content": "import mailbox\nimport os\nimport sys\n\nimport mailpile.mailboxes\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import UnorderedPicklable\n\n\nclass MailpileMailbox(UnorderedPicklable(mailbox.Maildir, editable=True)):\n \"\"\"A Maildir class that supports pickling and a few mailpile specifics.\"\"\"\n supported_platform = None\n\n @classmethod\n def parse_path(cls, config, fn, create=False, allow_empty=False):\n if (((cls.supported_platform is None) or\n (cls.supported_platform == sys.platform[:3].lower())) and\n ((os.path.isdir(fn) and\n os.path.exists(os.path.join(fn, 'cur'))) or\n (create and not os.path.exists(fn)))):\n return (fn, )\n raise ValueError('Not a Maildir: %s' % fn)\n\n def _refresh(self):\n with self._lock:\n mailbox.Maildir._refresh(self)\n # Dotfiles are not mail. Ignore them.\n for t in [k for k in self._toc.keys() if k.startswith('.')]:\n del self._toc[t]\n\n def __unicode__(self):\n return _(\"Maildir at %s\") % self._path\n\n def _describe_msg_by_ptr(self, msg_ptr):\n return _(\"e-mail in file %s\") % self._lookup(msg_ptr[MBX_ID_LEN:])\n\n def get_metadata_keywords(self, toc_id):\n subdir, name = os.path.split(self._lookup(toc_id))\n if self.colon in name:\n flags = name.split(self.colon)[-1]\n if flags[:2] == '2,':\n return ['%s:maildir' % c for c in flags[2:]]\n return []\n\n\nmailpile.mailboxes.register(25, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/maildirwin.py", "content": "import mailpile.mailboxes\nimport mailpile.mailboxes.maildir as maildir\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nclass MailpileMailbox(maildir.MailpileMailbox):\n \"\"\"A Maildir class for Windows (using ! instead of : in filenames)\"\"\"\n supported_platform = 'win'\n colon = '!'\n\n\nmailpile.mailboxes.register(20, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/mbox.py", "content": "from __future__ import print_function\nimport errno\nimport mailbox\nimport os\nimport re\nimport threading\nimport time\nimport traceback\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.index.mailboxes import MailboxIndex\nfrom mailpile.mailboxes import MBX_ID_LEN, NoSuchMailboxError\nfrom mailpile.util import *\n\n\nclass MboxIndex(MailboxIndex):\n pass\n\n\nclass MailpileMailbox(mailbox.mbox):\n \"\"\"A mbox class that supports pickling and a few mailpile specifics.\"\"\"\n RE_STATUS = re.compile(\n '^(X-)?Status:\\s*\\S+', flags=re.IGNORECASE|re.MULTILINE)\n\n @classmethod\n def parse_path(cls, config, fn, create=False, allow_empty=False):\n try:\n firstline = open(fn, 'r').readline()\n if firstline.startswith('From '):\n return (fn, )\n if (allow_empty or create) and not firstline:\n return (fn, )\n except:\n if create and os.path.exists(fn):\n return (fn, )\n raise ValueError('Not an mbox: %s' % fn)\n\n def __init__(self, *args, **kwargs):\n self._cs = {}\n mailbox.mbox.__init__(self, *args, **kwargs)\n self.editable = False\n self.is_local = False\n self._last_updated = 0\n self._mtime = 0\n self._index = None\n self._save_to = None\n self._encryption_key_func = lambda: None\n self._decryption_key_func = lambda: None\n self._lock = MboxRLock()\n\n def __enter__(self, *args, **kwargs):\n self._lock.acquire()\n self.lock()\n return self\n\n def __exit__(self, *args, **kwargs):\n self.unlock()\n self._lock.release()\n\n def __unicode__(self):\n return _(\"Unix mbox at %s\") % self._path\n\n def describe_msg_by_ptr(self, msg_ptr):\n try:\n parts, start, length = self._parse_ptr(msg_ptr)\n return _(\"message at bytes %d..%d\") % (start, start + length)\n except KeyError:\n return _(\"message not found in mailbox\")\n\n def _get_fd(self):\n return open(self._path, 'rb+')\n\n def __setstate__(self, dict):\n self.__dict__.update(dict)\n self._lock = MboxRLock()\n self.is_local = False\n with self._lock:\n self._save_to = None\n self._encryption_key_func = lambda: None\n self._decryption_key_func = lambda: None\n try:\n if not os.path.exists(self._path):\n raise NoSuchMailboxError(self._path)\n self._file = self._get_fd()\n except IOError as e:\n if e.errno == errno.ENOENT:\n raise NoSuchMailboxError(self._path)\n elif e.errno == errno.EACCES:\n self._file = self._get_fd()\n else:\n raise\n self.update_toc()\n\n def __getstate__(self):\n odict = self.__dict__.copy()\n # Pickle can't handle function objects.\n for dk in ('_save_to', '_index', '_last_updated',\n '_encryption_key_func', '_decryption_key_func',\n '_file', '_lock', 'parsed'):\n if dk in odict:\n del odict[dk]\n return odict\n\n def last_updated(self):\n return self._last_updated\n\n def keys(self):\n self.update_toc()\n return mailbox.mbox.keys(self)\n\n def toc_values(self):\n self.update_toc()\n return self._toc.values()\n\n def update_toc(self):\n fd = self._get_fd()\n fd.seek(0, 2)\n cur_length = fd.tell()\n cur_mtime = os.path.getmtime(self._path)\n try:\n if (self._file_length == cur_length and\n self._mtime == cur_mtime):\n return\n except (NameError, AttributeError):\n pass\n\n with self._lock:\n fd.seek(0)\n self._next_key = 0\n self._toc = {}\n self._cs = {}\n data = ''\n start = None\n len_nl = 1\n while (cur_length > 0):\n self._last_updated = time.time()\n line_pos = fd.tell()\n line = fd.readline()\n if line.startswith('From '):\n if start is not None:\n len_nl = ('\\r' == line[-2]) and 2 or 1\n cs4k = self.get_msg_cs4k(0, 0, data[:-len_nl])\n self._toc[self._next_key] = (start, line_pos - len_nl)\n self._cs[cs4k] = self._next_key\n self._cs[self._next_key] = cs4k\n self._next_key += 1\n start = line_pos\n data = line\n elif line == '':\n if (start is not None) and (start != line_pos):\n cs4k = self.get_msg_cs4k(0, 0, data[:-len_nl])\n self._toc[self._next_key] = (start, line_pos - len_nl)\n self._cs[cs4k] = self._next_key\n self._cs[self._next_key] = cs4k\n self._next_key += 1\n break\n elif len(data) < (4096 + len_nl):\n data += line\n self._file = fd\n self._file_length = fd.tell()\n self._mtime = cur_mtime\n self.save(None)\n\n def _generate_toc(self):\n self.update_toc()\n\n def __setitem__(self, *args, **kwargs):\n with self._lock:\n mailbox.mbox.__setitem__(self, *args, **kwargs)\n\n def __delitem__(self, *args, **kwargs):\n with self._lock:\n mailbox.mbox.__delitem__(self, *args, **kwargs)\n\n def save(self, session=None, to=None, pickler=None):\n if to and pickler:\n self._save_to = (pickler, to)\n if self._save_to and len(self) > 0:\n with self._lock:\n pickler, fn = self._save_to\n if session:\n session.ui.mark(_('Saving %s state to %s') % (self, fn))\n pickler(self, fn)\n\n def _locked_flush_without_tempfile(self):\n \"\"\"Dangerous, but we need this for /var/mail/USER on many Linuxes\"\"\"\n with open(self._path, 'rb+') as new_file:\n new_toc = {}\n for key in sorted(self._toc.keys()):\n start, stop = self._toc[key]\n new_start = new_file.tell()\n while True:\n buf = self._file.read(min(4096, stop-self._file.tell()))\n if buf == '':\n break\n new_file.write(buf)\n new_toc[key] = (new_start, new_file.tell())\n new_file.truncate()\n self._file.seek(0, 0)\n self._toc = new_toc\n self._pending = False\n self._pending_sync = False\n\n def flush(self, *args, **kwargs):\n with self._lock:\n self._last_updated = time.time()\n try:\n if kwargs.get('in_place', False):\n self._locked_flush_without_tempfile()\n else:\n mailbox.mbox.flush(self, *args, **kwargs)\n except OSError:\n if '_create_temporary' in traceback.format_exc():\n self._locked_flush_without_tempfile()\n else:\n raise\n self._last_updated = time.time()\n\n def clear(self, *args, **kwargs):\n with self._lock:\n mailbox.mbox.clear(self, *args, **kwargs)\n\n def get_msg_size(self, toc_id):\n try:\n with self._lock:\n # Note: This is 1 byte less than the TOC measures, because\n # the final newline is ommitted. The From line is\n # included though.\n start, stop = self._toc[toc_id]\n return (stop - start)\n except (IndexError, KeyError, IndexError, TypeError):\n return 0\n\n def get_metadata_keywords(self, toc_id):\n # In an mbox, all metadata is in the message headers.\n return []\n\n def set_metadata_keywords(self, *args, **kwargs):\n pass\n\n def get_index(self, config, mbx_mid=None):\n with self._lock:\n if self._index is None:\n self._index = MboxIndex(config, self, mbx_mid=mbx_mid)\n return self._index\n\n def get_msg_cs(self, start, cs_size, max_length, chars=4, data=None):\n \"\"\"Generate a checksum of a given length, ignoring Status headers.\"\"\"\n if data is None:\n if start is None:\n raise IOError('No data found (start=None)')\n with self._lock:\n fd = self._file\n fd.seek(start, 0)\n data = fd.read(min(cs_size, max_length))\n if data == '':\n raise IOError('No data found at %s:%s'\n % (start, max_length))\n elif len(data) >= cs_size:\n data = data[:cs_size]\n return b64w(sha1b64(\n re.sub(self.RE_STATUS, 'Status: ?', data))[:chars])\n\n def get_msg_cs4k(self, start, max_length, data=None):\n \"\"\"A 48-bit (6*8) checksum of the first 4k of message data.\"\"\"\n return self.get_msg_cs(start, 4096, max_length, chars=8, data=data)\n\n def get_msg_cs80b(self, start, max_length, data=None):\n \"\"\"A 24-bit (6*4) checksum of the first 80 bytes of message data.\"\"\"\n return self.get_msg_cs(start, 80, max_length, data=data)\n\n def get_msg_ptr(self, mboxid, toc_id, data=None):\n with self._lock:\n msg_start = self._toc[toc_id][0]\n msg_size = self.get_msg_size(toc_id)\n if (toc_id in self._cs) and (data is None):\n msg_cs = self._cs[toc_id]\n else:\n msg_cs = self.get_msg_cs4k(msg_start, msg_size, data=data)\n return '%s%s:%s:%s' % (\n mboxid, b36(msg_start), b36(msg_size), msg_cs)\n\n def _parse_ptr(self, msg_ptr):\n parts = msg_ptr[MBX_ID_LEN:].split(':')\n start = int(parts[0], 36)\n length = int(parts[1], 36)\n if len(parts) > 2:\n if parts[2] in self._cs:\n start, end = self._toc[self._cs[parts[2]]]\n length = end - start\n return parts, start, length\n\n def _verify_ptr_checksums(self, msg_ptr, start, ignored_fd):\n \"\"\"Check whether the msg_ptr checksums match the data at [start].\"\"\"\n with self._lock:\n parts, ignored_start, length = self._parse_ptr(msg_ptr)\n cs80b = self.get_msg_cs80b(start, length)\n if len(parts) > 2:\n cs4k = self.get_msg_cs4k(start, length)\n cs = parts[2]\n if (cs4k != cs and cs80b != cs):\n return False\n return True\n\n def _possible_message_locations(self, msg_ptr, max_locations=15):\n \"\"\"Yield possible locations for messages of a given size.\"\"\"\n with self._lock:\n parts, pstart, length = self._parse_ptr(msg_ptr)\n\n # This is where it is SUPPOSED to be, always check that first.\n starts = [pstart]\n\n # Extend the list with other messages of the right size.\n # We accept two lengths, because there were off-by-one errors\n # in older versions of Mailpile. :-(\n starts.extend(sorted([\n b for b, e in self.toc_values()\n if length in (e-b, e-b+1) and b != pstart]))\n\n # Yield up to max_locations positions\n for i, start in enumerate(starts[:max_locations]):\n yield (start, length)\n\n def _get_SSLP_by_ptr(self, msg_ptr, verifier=None, from_=False):\n if verifier is None:\n verifier = self._verify_ptr_checksums\n tries = []\n length = None\n for from_start, length in self._possible_message_locations(msg_ptr):\n # We duplicate the file descriptor here, in case other threads\n # are accessing the same mailbox and moving it around, or in\n # case we have multiple PartialFile objects in flight at once.\n tries.append(str(from_start))\n try:\n start = from_start\n stop = from_start + length\n fd = self._get_fd()\n if not from_:\n fd.seek(start)\n length -= len(fd.readline())\n start = fd.tell()\n pf = mailbox._PartialFile(fd, start, stop)\n if verifier(msg_ptr, from_start, pf):\n return (from_start, start, length, pf)\n except IOError:\n pass\n err = '%s: %s %s@%s' % (\n _('Message not found'), msg_ptr, length, '/'.join(tries))\n raise IOError(err)\n\n def update(self, *args, **kwargs):\n with self._lock:\n self._cs = {} # FIXME\n return mailbox.mbox.update(self, *args, **kwargs)\n\n def discard(self, *args, **kwargs):\n with self._lock:\n self._cs = {} # FIXME\n return mailbox.mbox.discard(self, *args, **kwargs)\n\n def remove(self, *args, **kwargs):\n with self._lock:\n self._cs = {} # FIXME\n return mailbox.mbox.remove(self, *args, **kwargs)\n\n def get_file_by_ptr(self, msg_ptr, verifier=None, from_=False):\n with self._lock:\n from_start, start, length, pfile = self._get_SSLP_by_ptr(\n msg_ptr, verifier=verifier, from_=from_)\n return pfile\n\n def remove_by_ptr(self, msg_ptr):\n with self._lock:\n from_start, start, length, pfile = self._get_SSLP_by_ptr(msg_ptr)\n keys = [k for k in self._toc if self._toc[k][0] == from_start]\n if keys:\n return self.remove(keys[0])\n raise KeyError('Not found: %s' % msg_ptr)\n\n def get_bytes(self, toc_id, *args, **kwargs):\n with self._lock:\n return self.get_file(toc_id, *args, **kwargs).read()\n\n def get_file(self, *args, **kwargs):\n with self._lock:\n return mailbox.mbox.get_file(self, *args, **kwargs)\n\n\nif __name__ == \"__main__\":\n import tempfile, time, sys\n verbose = ('-v' in sys.argv) or ('--verbose' in sys.argv)\n wait = ('-w' in sys.argv) or ('--wait' in sys.argv)\n\n MSG_TEMPLATE = \"\"\"\\\nFrom bre@mailpile.is Mon Jan 1 08:14:00 2018\nReturn-Path: \nSubject: %(subject)s\nMessage-ID: <%(msgid)s>\nContent-Length: %(length)s\n\n%(content)s\"\"\"\n\n problems = tests = 0\n with tempfile.NamedTemporaryFile() as tf:\n lengths = []\n for count in range(0, 35):\n body = ''.join([\n 'Hello world, this is a message!\\n'\n ] * ((27 * (100-count)) % 1230))\n message = (MSG_TEMPLATE % {\n 'subject': 'Test message #%d' % count,\n 'msgid': '%d@example.com' % count,\n 'length': len(body),\n 'content': body})\n lengths.append(len(message))\n tf.write(message)\n tf.write(\"\\n\")\n tf.flush()\n if verbose or wait:\n print('Temporary mailbox in: %s' % tf.name)\n if wait:\n raw_input('Press ENTER to continue...')\n\n pmbx = mailbox.mbox(tf.name)\n mmbx = MailpileMailbox(tf.name)\n ptrs = []\n for i, key in enumerate(mmbx.keys()):\n msg_ptr = mmbx.get_msg_ptr('0000', key)\n o_size = lengths[i]\n c_size = mmbx.get_msg_size(key)\n f_size = len(mmbx.get_bytes(key, from_=True))\n f2size = len(mmbx.get_file_by_ptr(msg_ptr, from_=True).read())\n result = 'ok' if (o_size == c_size == f_size == f2size) else 'BAD'\n if verbose or result != 'ok':\n print(\"%-3.3s [%s/%s/%s] %s ?= %s ?= %s ?= %s\" % (\n result, i, key, msg_ptr, o_size, c_size, f_size, f2size))\n if result != 'ok':\n problems += 1\n tests += 1\n ptrs.append([msg_ptr, f2size])\n\n # Remove some messages, bypassing MailpileMailbox\n deletions = [0, 5, 10, 15, 34]\n for d in reversed(sorted(deletions)):\n del pmbx[d]\n pmbx.flush()\n\n # Remove a message using MailpileMailbox\n try:\n tests += 1\n deletions.append(1)\n mmbx.remove_by_ptr(ptrs[1][0])\n mmbx.flush()\n except KeyError:\n problems += 1\n\n for i, (msg_ptr, f2size) in enumerate(ptrs):\n tests += 1\n if i in deletions:\n try:\n mmbx.get_file_by_ptr(msg_ptr, from_=True).read()\n problems += 1\n print('BAD Found deleted message %s' % msg_ptr)\n except IOError:\n if verbose:\n print('ok IOError on message %s' % msg_ptr)\n continue\n f3size = len(mmbx.get_file_by_ptr(msg_ptr, from_=True).read())\n if (f2size != f3size):\n problems += 1\n print('BAD Message %s: wrong size in new location' % msg_ptr)\n elif verbose:\n print('ok Message %s found in new location' % msg_ptr)\n\n # This is formatted to look like doctest results...\n print('TestResults(failed=%d, attempted=%d)' % (problems, tests))\n if wait:\n raw_input('Tests finished. Press ENTER to clean up...')\n\n if problems:\n sys.exit(1)\nelse:\n import mailpile.mailboxes\n mailpile.mailboxes.register(90, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/pop3.py", "content": "from __future__ import print_function\ntry:\n import cStringIO as StringIO\nexcept ImportError:\n import StringIO\n\nimport poplib\nimport socket\nimport ssl\nimport time\nfrom mailbox import Mailbox, Message\n\nimport mailpile.mailboxes\nfrom mailpile.conn_brokers import Master as ConnBroker\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import UnorderedPicklable\nfrom mailpile.util import *\n\n\nclass wrappable_POP3_SSL(poplib.POP3_SSL):\n \"\"\"\n Override the default poplib.POP3_SSL init to use socket.create_connection\n \"\"\"\n def __init__(self, host,\n port=poplib.POP3_SSL_PORT, keyfile=None, certfile=None,\n timeout=socket._GLOBAL_DEFAULT_TIMEOUT):\n self.host = host\n self.port = port\n self.keyfile = keyfile\n self.certfile = certfile\n self.buffer = \"\"\n self.sock = socket.create_connection((host, port), timeout)\n self.file = self.sock.makefile('rb')\n self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile)\n self._debugging = 0\n self.welcome = self._getresp()\n\n\nclass UnsupportedProtocolError(Exception):\n pass\n\n\nclass POP3Mailbox(Mailbox):\n \"\"\"\n Basic implementation of POP3 Mailbox.\n \"\"\"\n def __init__(self, host,\n user=None, password=None, auth_type='password',\n use_ssl=True, port=None, debug=False, conn_cls=None,\n session=None):\n \"\"\"Initialize a Mailbox instance.\"\"\"\n Mailbox.__init__(self, '/')\n self.host = host\n self.user = user\n self.password = password\n self.auth_type = auth_type\n self.use_ssl = use_ssl\n self.port = port\n self.debug = debug\n self.conn_cls = conn_cls\n self.session = session\n\n self._lock = MboxRLock()\n self._pop3 = None\n self._connect()\n\n def lock(self):\n pass\n\n def unlock(self):\n pass\n\n def _connect(self):\n with self._lock:\n if self._pop3:\n try:\n self._pop3.noop()\n return\n except poplib.error_proto:\n self._pop3 = None\n\n with ConnBroker.context(need=[ConnBroker.OUTGOING_POP3]):\n if self.conn_cls:\n self._pop3 = self.conn_cls(self.host, self.port or 110,\n timeout=120)\n self.secure = self.use_ssl\n elif self.use_ssl:\n self._pop3 = wrappable_POP3_SSL(self.host, self.port or 995,\n timeout=120)\n self.secure = True\n else:\n self._pop3 = poplib.POP3(self.host, self.port or 110,\n timeout=120)\n self.secure = False\n\n if hasattr(self._pop3, 'sock'):\n self._pop3.sock.settimeout(120)\n if self.debug:\n self._pop3.set_debuglevel(self.debug)\n\n self._keys = None\n try:\n if self.auth_type.lower() == 'oauth2':\n from mailpile.plugins.oauth import OAuth2\n token_info = OAuth2.GetFreshTokenInfo(self.session,\n self.user)\n if self.user and token_info and token_info.access_token:\n raise AccessError(\"FIXME: Do OAUTH2 Auth!\")\n else:\n raise AccessError()\n else:\n self._pop3.user(self.user)\n self._pop3.pass_(self.password.encode('utf-8'))\n except poplib.error_proto:\n raise AccessError()\n\n def _refresh(self):\n with self._lock:\n self._keys = None\n self.iterkeys()\n\n def __setitem__(self, key, message):\n \"\"\"Replace the keyed message; raise KeyError if it doesn't exist.\"\"\"\n raise NotImplementedError('Method must be implemented by subclass')\n\n def _get(self, key, _bytes=None):\n with self._lock:\n if key not in self.iterkeys():\n raise KeyError('Invalid key: %s' % key)\n\n self._connect()\n if _bytes is not None:\n lines = max(10, _bytes//30) # A wild guess!\n ok, lines, octets = self._pop3.top(self._km[key], lines)\n else:\n ok, lines, octets = self._pop3.retr(self._km[key])\n if not ok.startswith('+OK'):\n raise KeyError('Invalid key: %s' % key)\n\n # poplib is stupid in that it loses the linefeeds, so we need to\n # do some guesswork to bring them back to what the server provided.\n # If we don't do this jiggering, then sizes don't match up, which\n # could cause allocation bugs down the line.\n\n have_octets = sum(len(l) for l in lines)\n if octets == have_octets + len(lines):\n lines.append('')\n data = '\\n'.join(lines)\n elif octets == have_octets + 2*len(lines):\n lines.append('')\n data = '\\r\\n'.join(lines)\n elif octets == have_octets + len(lines) - 1:\n data = '\\n'.join(lines)\n elif octets == have_octets + 2*len(lines) - 2:\n data = '\\r\\n'.join(lines)\n else:\n raise ValueError('Length mismatch in message %s' % key)\n\n if _bytes is not None:\n return data[:_bytes]\n else:\n return data\n\n def get_message(self, key):\n \"\"\"Return a Message representation or raise a KeyError.\"\"\"\n return Message(self._get(key))\n\n def get_bytes(self, key, *args):\n \"\"\"Return a byte string representation or raise a KeyError.\"\"\"\n return self._get(key, *args)\n\n def get_file(self, key):\n \"\"\"Return a file-like representation or raise a KeyError.\"\"\"\n return StringIO.StringIO(self._get(key))\n\n def get_msg_size(self, key):\n with self._lock:\n self._connect()\n if key not in self.iterkeys():\n raise KeyError('Invalid key: %s' % key)\n ok, info, octets = self._pop3.list(self._km[key]).split()\n return int(octets)\n\n def remove(self, key):\n # FIXME: This is very inefficient if we are deleting multiple\n # messages at once.\n with self._lock:\n self._connect()\n if key not in self.iterkeys():\n raise KeyError('Invalid key: %s' % key)\n ok = self._pop3.dele(self._km[key])\n self._refresh()\n\n def stat(self):\n with self._lock:\n self._connect()\n return self._pop3.stat()\n\n def iterkeys(self):\n \"\"\"Return an iterator over keys.\"\"\"\n # Note: POP3 *without UIDL* is useless. We don't support it.\n with self._lock:\n if self._keys is None:\n self._connect()\n try:\n stat, key_list, octets = self._pop3.uidl()\n except poplib.error_proto:\n raise UnsupportedProtocolError()\n self._keys = [tuple(k.split(' ', 1)) for k in key_list]\n self._km = dict([reversed(k) for k in self._keys])\n return [k[1] for k in self._keys]\n\n def __contains__(self, key):\n \"\"\"Return True if the keyed message exists, False otherwise.\"\"\"\n return key in self.iterkeys()\n\n def __len__(self):\n \"\"\"Return a count of messages in the mailbox.\"\"\"\n return len(self.iterkeys())\n\n def flush(self):\n \"\"\"Write any pending changes to the disk.\"\"\"\n self.close()\n\n def close(self):\n \"\"\"Flush and close the mailbox.\"\"\"\n try:\n if self._pop3:\n self._pop3.quit()\n finally:\n self._pop3 = None\n self._keys = None\n\n\nclass MailpileMailbox(UnorderedPicklable(POP3Mailbox)):\n UNPICKLABLE = ['_pop3', '_debug']\n\n @classmethod\n def parse_path(cls, config, path, create=False, allow_empty=False):\n path = path.split('/')\n if path and path[0].lower() in ('pop:', 'pop3:',\n 'pop3_ssl:', 'pop3s:'):\n proto = path[0][:-1].lower()\n userpart, server = path[2].rsplit(\"@\", 1)\n user, password = userpart.rsplit(\":\", 1)\n if \":\" in server:\n server, port = server.split(\":\", 1)\n else:\n port = 995 if ('s' in proto) else 110\n\n # This is a hack for GMail\n if 'recent' in path[3:]:\n user = 'recent:' + user\n\n if not config:\n debug = False\n elif 'pop3' in config.sys.debug:\n debug = 99\n elif 'rescan' in config.sys.debug:\n debug = 1\n else:\n debug = False\n\n # WARNING: Order must match POP3Mailbox.__init__(...)\n return (server, user, password, 's' in proto, int(port), debug)\n raise ValueError('Not a POP3 url: %s' % path)\n\n def save(self, *args, **kwargs):\n # Do not save state locally\n pass\n\n\n##[ Test code follows ]#######################################################\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n\n class _MockPOP3(object):\n \"\"\"\n Base mock that pretends to be a poplib POP3 connection.\n\n >>> pm = POP3Mailbox('localhost', user='bad', conn_cls=_MockPOP3)\n Traceback (most recent call last):\n ...\n AccessError\n\n >>> pm = POP3Mailbox('localhost', user='a', password='b',\n ... conn_cls=_MockPOP3)\n >>> pm.stat()\n (2, 123456)\n\n >>> pm.iterkeys()\n ['evil', 'good']\n\n >>> 'evil' in pm, 'bogon' in pm\n (True, False)\n\n >>> [msg['subject'] for msg in pm]\n ['Msg 1', 'Msg 2']\n\n >>> pm.get_msg_size('evil'), pm.get_msg_size('good')\n (47, 51)\n\n >>> pm.get_bytes('evil')\n 'From: test@mailpile.is\\\\nSubject: Msg 1\\\\n\\\\nOh, hi!\\\\n'\n\n >>> pm.get_bytes('evil', 5)\n 'From:'\n\n >>> pm['invalid-key']\n Traceback (most recent call last):\n ...\n KeyError: ...\n \"\"\"\n TEST_MSG = ('From: test@mailpile.is\\r\\n'\n 'Subject: Msg N\\r\\n'\n '\\r\\n'\n 'Oh, hi!\\r\\n')\n DEFAULT_RESULTS = {\n 'user': lambda s, u: '+OK' if (u == 'a') else '-ERR',\n 'pass_': lambda s, u: '+OK Logged in.' if (u == 'b') else '-ERR',\n 'stat': (2, 123456),\n 'noop': '+OK',\n 'list_': lambda s: ('+OK 2 messages:',\n ['1 %d' % len(s.TEST_MSG.replace('\\r', '')),\n '2 %d' % len(s.TEST_MSG)], 0),\n 'uidl': ('+OK', ['1 evil', '2 good'], 0),\n 'retr': lambda s, m: ('+OK',\n s.TEST_MSG.replace('N', m).splitlines(),\n len(s.TEST_MSG)\n if m[0] == '2' else\n len(s.TEST_MSG.replace('\\r', ''))),\n 'top': lambda s, m, n: ('+OK',\n s.TEST_MSG.splitlines()[:n],\n len(''.join(s.TEST_MSG.splitlines(1)[:n]))),\n }\n RESULTS = {}\n\n def __init__(self, *args, **kwargs):\n def mkcmd(rval):\n def r(rv):\n if isinstance(rv, (str, unicode)) and rv[0] != '+':\n raise poplib.error_proto(rv)\n return rv\n\n def cmd(*args, **kwargs):\n if isinstance(rval, (str, unicode, list, tuple, dict)):\n return r(rval)\n else:\n return r(rval(self, *args, **kwargs))\n\n return cmd\n for cmd, rval in dict_merge(self.DEFAULT_RESULTS, self.RESULTS\n ).iteritems():\n self.__setattr__(cmd, mkcmd(rval))\n\n def list(self, which=None):\n msgs = self.list_()\n if which:\n return '+OK ' + msgs[1][1-int(which)]\n return msgs\n\n def __getattr__(self, attr):\n return self.__getattribute__(attr)\n\n class _MockPOP3_Without_UIDL(_MockPOP3):\n \"\"\"\n Mock that lacks the UIDL command.\n\n >>> pm = POP3Mailbox('localhost', user='a', password='b',\n ... conn_cls=_MockPOP3_Without_UIDL)\n >>> pm.iterkeys()\n Traceback (most recent call last):\n ...\n UnsupportedProtocolError\n \"\"\"\n RESULTS = {'uidl': '-ERR'}\n\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n\n if len(sys.argv) > 1:\n mbx = MailpileMailbox(*MailpileMailbox.parse_path(None, sys.argv[1]))\n print('Status is: %s' % (mbx.stat(), ))\n print('Downloading mail and listing subjects, hit CTRL-C to quit')\n for msg in mbx:\n print(msg['subject'])\n time.sleep(2)\n\nelse:\n mailpile.mailboxes.register(10, MailpileMailbox)\n" }, { "path": "mailpile/mailboxes/wervd.py", "content": "import email.generator\nimport email.message\nimport mailbox\nimport StringIO\nimport sys\n\nimport mailpile.mailboxes\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import UnorderedPicklable, MBX_ID_LEN\nfrom mailpile.crypto.streamer import *\nfrom mailpile.util import safe_remove\n\n\nclass MailpileMailbox(UnorderedPicklable(mailbox.Maildir, editable=True)):\n \"\"\"A Maildir class that supports pickling and a few mailpile specifics.\"\"\"\n supported_platform = None\n colon = '!' # Works on both Windows and Unix\n\n# FIXME: Copies were part of the original WERVD spec, to compensate for\n# the additional fragility of encrypted data. This hasn't been\n# implemented however, and while SSDs are expensive it is not\n# obvious that doubling (or tripling...) the storage requirements\n# for all e-mail is a cost folks are willing to pay for never\n# losing a message to the occaisional bitflips. So this is all\n# commented out at the moment. Revisit?\n# MAX_COPIES = 5\n\n @classmethod\n def parse_path(cls, config, fn, create=False, allow_empty=False):\n if (((cls.supported_platform is None) or\n (cls.supported_platform == sys.platform[:3].lower())) and\n ((os.path.isdir(fn) and\n os.path.exists(os.path.join(fn, 'cur')) and\n os.path.exists(os.path.join(fn, 'wervd.ver'))) or\n (create and not os.path.exists(fn)))):\n return (fn, )\n raise ValueError('Not a Mailpile Maildir: %s' % fn)\n\n def __init2__(self, *args, **kwargs):\n open(os.path.join(self._path, 'wervd.ver'), 'w+b').write('0')\n\n def __unicode__(self):\n return _(\"Mailpile mailbox at %s\") % self._path\n\n def _describe_msg_by_ptr(self, msg_ptr):\n return _(\"e-mail in file %s\") % self._lookup(msg_ptr[MBX_ID_LEN:])\n\n# FIXME: Copies\n# def _copy_paths(self, where, key, copies):\n# for cpn in range(1, copies):\n# yield os.path.join(self._path, where, '%s.%s' % (key, cpn))\n\n def remove(self, key):\n with self._lock:\n fn = os.path.join(self._path, self._lookup(key))\n del self._toc[key]\n safe_remove(fn)\n\n# FIXME: Copies\n# # Also remove all the copies of this message!\n# key = os.path.basename(fn)\n# for where in ('cur', 'new', 'tmp'):\n# for copy_fn in self._copy_paths(where, key, self.MAX_COPIES):\n# if os.path.exists(copy_fn):\n# safe_remove(copy_fn)\n# else:\n# break\n\n def _refresh(self):\n with self._lock:\n mailbox.Maildir._refresh(self)\n # WERVD mail names don't have dots in them\n for t in [k for k in self._toc.keys() if '.' in k]:\n del self._toc[t]\n safe_remove() # Try to remove any postponed removals\n\n def _get_fd(self, key):\n with self._lock:\n fn = os.path.join(self._path, self._lookup(key))\n mep_key = self._decryption_key_func()\n fd = open(fn, 'rb')\n if mep_key:\n fd = DecryptingStreamer(fd, mep_key=mep_key,\n name='WERVD(%s)' % fn)\n return fd\n\n def get_message(self, key):\n \"\"\"Return a Message representation or raise a KeyError.\"\"\"\n with self._lock:\n with self._get_fd(key) as fd:\n if self._factory:\n return self._factory(fd)\n else:\n return mailbox.MaildirMessage(fd)\n\n def get_string(self, key):\n with self._lock:\n with self._get_fd(key) as fd:\n return fd.read()\n\n def get_file(self, key):\n with self._lock:\n return StringIO.StringIO(self.get_string(key))\n\n def get_metadata_keywords(self, toc_id):\n subdir, name = os.path.split(self._lookup(toc_id))\n if self.colon in name:\n flags = name.split(self.colon)[-1]\n if flags[:2] == '2,':\n return ['%s:maildir' % c for c in flags[2:]]\n return []\n\n def set_metadata_keywords(self, toc_id, kws):\n with self._lock:\n old_fpath = self._lookup(toc_id)\n new_fpath = old_fpath.rsplit(self.colon, 1)[0]\n\n flags = ''.join(sorted([k[0] for k in kws]))\n if flags:\n new_fpath += '%s2,%s' % (self.colon, flags)\n if new_fpath != old_fpath:\n os.rename(os.path.join(self._path, old_fpath),\n os.path.join(self._path, new_fpath))\n self._toc[toc_id] = new_fpath\n\n def add(self, message):\n \"\"\"Add message and return assigned key.\"\"\"\n key = self._encryption_key_func()\n es = None\n try:\n tmpdir = os.path.join(self._path, 'tmp')\n if not os.path.exists(tmpdir):\n os.mkdir(tmpdir, 0o700)\n if key:\n es = EncryptingStreamer(key,\n dir=tmpdir, name='WERVD',\n delimited=False)\n else:\n es = ChecksummingStreamer(dir=tmpdir, name='WERVD')\n self._dump_message(message, es)\n es.finish()\n\n # We are using the MAC to detect file system corruption, not in a\n # security context - so using as little as 40 bits should be fine.\n saved = False\n key = None\n outer_mac = es.outer_mac_sha256()\n for l in range(10, len(outer_mac)):\n key = outer_mac[:l]\n fn = os.path.join(self._path, 'new', key)\n if not os.path.exists(fn):\n es.save(fn)\n saved = self._toc[key] = os.path.join('new', key)\n break\n if not saved:\n raise mailbox.ExternalClashError(_('Could not find a filename '\n 'for the message.'))\n\n# FIXME: Copies\n# for fn in self._copy_paths('new', key, copies):\n# with mailbox._create_carefully(fn) as ofd:\n# es.save_copy(ofd)\n\n return key\n finally:\n if es is not None:\n es.close()\n\n def _dump_message(self, message, target):\n if isinstance(message, email.message.Message):\n gen = email.generator.Generator(target, False, 0)\n gen.flatten(message)\n elif isinstance(message, str):\n target.write(message)\n else:\n raise TypeError(_('Invalid message type: %s') % type(message))\n\n def __setitem__(self, key, message):\n raise IOError(_('Mailbox messages are immutable'))\n\n\nmailpile.mailboxes.register(15, MailpileMailbox)\n" }, { "path": "mailpile/mailutils/__init__.py", "content": "# vim: set fileencoding=utf-8 :\n#\nMBX_ID_LEN = 4 # 4x36 == 1.6 million mailboxes\n\n\ndef FormatMbxId(n):\n if not isinstance(n, (str, unicode)):\n n = b36(n)\n if len(n) > MBX_ID_LEN:\n raise ValueError(_('%s is too large to be a mailbox ID') % n)\n return ('0000' + n).lower()[-MBX_ID_LEN:]\n\n\nclass NotEditableError(ValueError):\n pass\n\n\nclass NoFromAddressError(ValueError):\n pass\n\n\nclass NoRecipientError(ValueError):\n pass\n\n\nclass InsecureSmtpError(IOError):\n def __init__(self, msg, details=None):\n IOError.__init__(self, msg)\n self.error_info = details or {}\n\n\nclass NoSuchMailboxError(OSError):\n pass\n" }, { "path": "mailpile/mailutils/addresses.py", "content": "# vim: set fileencoding=utf-8 :\n#\nfrom __future__ import print_function\nimport base64\nimport copy\nimport quopri\nimport re\n\nfrom mailpile.util import *\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.vcard import AddressInfo\n\n\nclass AddressHeaderParser(list):\n \"\"\"\n This is a class which tries very hard to interpret the From:, To:\n and Cc: lines found in real-world e-mail and make sense of them.\n\n The general strategy of this parser is to:\n 1. parse header data into tokens\n 2. group tokens together into address + name constructs.\n\n And optionaly,\n 3. normalize each group to a standard format\n\n In practice, we do this in multiple passes: first a strict pass where\n we try to parse things semi-sensibly, followed by fuzzier heuristics.\n\n Ideally, if folks format things correctly we should parse correctly.\n But if that fails, there are are other passes where we try to cope\n with various types of weirdness we've seen in the wild. The wild can\n be pretty wild.\n\n This parser is NOT (yet) fully RFC2822 compliant - in particular it\n will get confused by nested comments (see FIXME in tests below).\n\n The normalization will take pains to ensure that < and , are never\n present inside a name/comment (even if legal), to make life easier\n for lame parsers down the line.\n\n Examples:\n\n >>> ahp = AddressHeaderParser(AddressHeaderParser.TEST_HEADER_DATA)\n >>> ai = ahp[1]\n >>> ai.fn\n u'Bjarni'\n >>> ai.address\n u'bre@klaki.net'\n >>> ahpn = ahp.normalized_addresses()\n >>> (ahpn == ahp.TEST_EXPECT_NORMALIZED_ADDRESSES) or ahpn\n True\n\n >>> AddressHeaderParser('Weird email@somewhere.com Header').normalized()\n u'\"Weird Header\" '\n\n >>> ai = AddressHeaderParser(unicode_data=ahp.TEST_UNICODE_DATA)\n >>> ai[0].fn\n u'Bjarni R\\\\xfanar'\n >>> ai[0].fn == ahp.TEST_UNICODE_NAME\n True\n >>> ai[0].address\n u'b@c.x'\n \"\"\"\n\n TEST_UNICODE_DATA = u'Bjarni R\\xfanar '\n TEST_UNICODE_NAME = u'Bjarni R\\xfanar'\n TEST_HEADER_DATA = \"\"\"\n bre@klaki.net ,\n bre@klaki.net Bjarni ,\n bre@klaki.net bre@klaki.net,\n bre@klaki.net (bre@notmail.com),\n \"\" ,\n =?utf-8?Q?=3Cbre@notmail.com=3E?= ,\n bre@klaki.net ((nested) bre@notmail.com comment),\n (FIXME: (nested) bre@wrongmail.com parser breaker) bre@klaki.net,\n undisclosed-recipients-gets-ignored:,\n Bjarni [mailto:bre@klaki.net],\n \"This is a key test\" ,\n bre@klaki.net (Bjarni Runar Einar's son);\n Bjarni =?iso-8859-1?Q??=is bre @klaki.net,\n Bjarni =?iso-8859-1?Q?Runar?=Einarsson<' bre'@ klaki.net>,\n \"Einarsson, Bjarni\" ,\n =?iso-8859-1?Q?Lonia_l=F6gmannsstofa?= ,\n \"Bjarni @ work\" ,\n \"\"\"\n TEST_EXPECT_NORMALIZED_ADDRESSES = [\n '',\n '\"Bjarni\" ',\n '\"bre@klaki.net\" ',\n '\"bre@notmail.com\" ',\n '=?utf-8?Q?=3Cbre@notmail.com=3E?= ',\n '=?utf-8?Q?=3Cbre@notmail.com=3E?= ',\n '\"(nested bre@notmail.com comment)\" ',\n '\"(FIXME: nested parser breaker) bre@klaki.net\" ',\n '\"Bjarni\" ',\n '\"This is a key test\" ',\n '\"Bjarni Runar Einar\\\\\\'s son\" ',\n '\"Bjarni is\" ',\n '\"Bjarni Runar Einarsson\" ',\n '=?utf-8?Q?Einarsson=2C_Bjarni?= ',\n '=?utf-8?Q?Lonia_l=C3=B6gmannsstofa?= ',\n '\"Bjarni @ work\" ']\n\n # Escaping and quoting\n TXT_RE_QUOTE = '=\\\\?([^\\\\?\\\\s]+)\\\\?([QqBb])\\\\?([^\\\\?\\\\s]*)\\\\?='\n TXT_RE_QUOTE_NG = TXT_RE_QUOTE.replace('(', '(?:')\n RE_ESCAPES = re.compile('\\\\\\\\([\\\\\\\\\"\\'])')\n RE_QUOTED = re.compile(TXT_RE_QUOTE)\n RE_SHOULD_ESCAPE = re.compile('([\\\\\\\\\"\\'])')\n RE_SHOULD_QUOTE = re.compile('[^a-zA-Z0-9()\\.:/_ \\'\"+@-]')\n\n # This is how we normally break a header line into tokens\n RE_TOKENIZER = re.compile('(<[^<>]*>' # \n '|\\\\([^\\\\(\\\\)]*\\\\)' # (stuff)\n '|\\\\[[^\\\\[\\\\]]*\\\\]' # [stuff]\n '|\"(?:\\\\\\\\\\\\\\\\|\\\\\\\\\"|[^\"])*\"' # \"stuff\"\n \"|'(?:\\\\\\\\\\\\\\\\|\\\\\\\\'|[^'])*'\" # 'stuff'\n '|' + TXT_RE_QUOTE_NG + # =?stuff?=\n '|,' # ,\n '|;' # ;\n '|\\\\s+' # white space\n '|[^\\\\s;,]+' # non-white space\n ')')\n\n # Where to insert spaces to help the tokenizer parse bad data\n RE_MUNGE_TOKENSPACERS = (re.compile('(\\S)(<)'), re.compile('(\\S)(=\\\\?)'))\n\n # Characters to strip aware entirely when tokenizing munged data\n RE_MUNGE_TOKENSTRIPPERS = (re.compile('[<>\"]'),)\n\n # This is stuff we ignore (undisclosed-recipients, etc)\n RE_IGNORED_GROUP_TOKENS = re.compile('(?i)undisclosed')\n\n # Things we strip out to try and un-mangle e-mail addresses when\n # working with bad data.\n RE_MUNGE_STRIP = re.compile('(?i)(?:\\\\bmailto:|[\\\\s\"\\']|\\?$)')\n\n # This a simple regular expression for detecting e-mail addresses.\n RE_MAYBE_EMAIL = re.compile('^[^()<>@,;:\\\\\\\\\"\\\\[\\\\]\\\\s\\000-\\031]+'\n '@[a-zA-Z0-9_\\\\.-]+(?:#[A-Za-z0-9]+)?$')\n\n # We try and interpret non-ascii data as a particular charset, in\n # this order by default. Should be overridden whenever we have more\n # useful info from the message itself.\n DEFAULT_CHARSET_ORDER = ('iso-8859-1', 'utf-8')\n\n def __init__(self,\n data=None, unicode_data=None, charset_order=None, **kwargs):\n self.charset_order = charset_order or self.DEFAULT_CHARSET_ORDER\n self._parse_args = kwargs\n if data is None and unicode_data is None:\n self._reset(**kwargs)\n elif data is not None:\n self.parse(data)\n else:\n self.charset_order = ['utf-8']\n self.parse(unicode_data.encode('utf-8'))\n\n def _reset(self, _raw_data=None, strict=False, _raise=False):\n self._raw_data = _raw_data\n self._tokens = []\n self._groups = []\n self[:] = []\n\n def parse(self, data):\n return self._parse(data, **self._parse_args)\n\n def _parse(self, data, strict=False, _raise=False):\n self._reset(_raw_data=data)\n\n # 1st pass, strict\n try:\n self._tokens = self._tokenize(self._raw_data)\n self._groups = self._group(self._tokens)\n self[:] = self._find_addresses(self._groups,\n _raise=(not strict))\n return self\n except ValueError:\n if strict and _raise:\n raise\n if strict:\n return self\n\n # 2nd & 3rd passes; various types of sloppy\n for _pass in ('2', '3'):\n try:\n self._tokens = self._tokenize(self._raw_data, munge=_pass)\n self._groups = self._group(self._tokens, munge=_pass)\n self[:] = self._find_addresses(self._groups,\n munge=_pass,\n _raise=_raise)\n return self\n except ValueError:\n if _pass == '3' and _raise:\n raise\n return self\n\n def unquote(self, string, charset_order=None):\n def uq(m):\n cs, how, data = m.group(1), m.group(2), m.group(3)\n if how in ('b', 'B'):\n try:\n data = base64.b64decode(''.join(data.split())+'===')\n except TypeError:\n print('FAILED TO B64DECODE: %s' % data)\n else:\n data = quopri.decodestring(data, header=True)\n try:\n return data.decode(cs)\n except LookupError:\n return data.decode('iso-8859-1') # Always works\n\n for cs in charset_order or self.charset_order:\n try:\n string = string.decode(cs)\n break\n except UnicodeDecodeError:\n pass\n\n return re.sub(self.RE_QUOTED, uq, string)\n\n @classmethod\n def unescape(self, string):\n return re.sub(self.RE_ESCAPES, lambda m: m.group(1), string)\n\n @classmethod\n def escape(self, strng):\n return re.sub(self.RE_SHOULD_ESCAPE, lambda m: '\\\\'+m.group(0), strng)\n\n @classmethod\n def quote(self, strng):\n if re.search(self.RE_SHOULD_QUOTE, strng):\n enc = quopri.encodestring(strng.encode('utf-8'), False,\n header=True)\n enc = enc.replace('<', '=3C').replace('>', '=3E')\n enc = enc.replace(',', '=2C')\n return '=?utf-8?Q?%s?=' % enc\n else:\n return '\"%s\"' % self.escape(strng)\n\n def _tokenize(self, string, munge=False):\n if munge:\n for ts in self.RE_MUNGE_TOKENSPACERS:\n string = re.sub(ts, '\\\\1 \\\\2', string)\n if munge == 3:\n for ts in self.RE_MUNGE_TOKENSTRIPPERS:\n string = re.sub(ts, '', string)\n return re.findall(self.RE_TOKENIZER, string)\n\n def _clean(self, token):\n if token[:1] in ('\"', \"'\"):\n if token[:1] == token[-1:]:\n return self.unescape(token[1:-1])\n elif token.startswith('[mailto:') and token[-1:] == ']':\n # Just convert [mailto:...] crap into a
\n return '<%s>' % token[8:-1]\n elif (token[:1] == '[' and token[-1:] == ']'):\n return token[1:-1]\n return token\n\n def _group(self, tokens, munge=False):\n groups = [[]]\n for token in tokens:\n token = token.strip()\n if token in (',', ';'):\n # Those tokens SHOULD separate groups, but we don't like to\n # create groups that have no e-mail addresses at all.\n if groups[-1]:\n if [g for g in groups[-1] if '@' in g]:\n groups.append([])\n continue\n # However, this stuff is just begging to be ignored.\n elif [g for g in groups[-1]\n if re.match(self.RE_IGNORED_GROUP_TOKENS, g)]:\n groups[-1] = []\n continue\n if token:\n groups[-1].append(self.unquote(self._clean(token)))\n if not groups[-1]:\n groups.pop(-1)\n return groups\n\n def _find_addresses(self, groups, **fa_kwargs):\n alist = [self._find_address(g, **fa_kwargs) for g in groups]\n return [a for a in alist if a]\n\n def _find_address(self, g, _raise=False, munge=False):\n if g:\n g = g[:]\n else:\n return []\n\n def email_at(i):\n for j in range(0, len(g)):\n if g[j][:1] == '(' and g[j][-1:] == ')':\n g[j] = g[j][1:-1]\n rest = ' '.join([g[j] for j in range(0, len(g))\n if (j != i) and g[j]\n ]).replace(' ,', ',').replace(' ;', ';')\n email, keys = g[i], None\n if '#' in email[email.index('@'):]:\n email, key = email.rsplit('#', 1)\n keys = [{'fingerprint': key}]\n return AddressInfo(email, rest.strip(), keys=keys)\n\n def munger(string):\n if munge:\n return re.sub(self.RE_MUNGE_STRIP, '', string)\n else:\n return string\n\n # If munging, look for email @domain.com in two parts, rejoin\n if munge:\n for i in range(0, len(g)):\n if i > 0 and i < len(g) and g[i][:1] == '@':\n g[i-1:i+1] = [g[i-1]+g[i]]\n elif i < len(g)-1 and g[i][-1:] == '@':\n g[i:i+2] = [g[i]+g[i+1]]\n\n # 1st, look for \n #\n # We search from the end, to make the algorithm stable in the case\n # that the name part also starts with a < (is that allowed?).\n #\n for i in reversed(range(0, len(g))):\n if g[i][:1] == '<' and g[i][-1:] == '>':\n maybemail = munger(g[i][1:-1])\n if re.match(self.RE_MAYBE_EMAIL, maybemail):\n g[i] = maybemail\n return email_at(i)\n\n # 2nd, look for bare email@domain.com\n for i in range(0, len(g)):\n maybemail = munger(g[i])\n if re.match(self.RE_MAYBE_EMAIL, maybemail):\n g[i] = maybemail\n return email_at(i)\n\n if _raise:\n raise ValueError('No email found in %s' % (g,))\n else:\n return None\n\n def addresses_list(self, with_keys=False):\n addresses = []\n for addr in self:\n m = addr.address\n if with_keys and addr.keys:\n m += \"#\" + addr.keys[0].get('fingerprint')\n addresses.append(m)\n return addresses\n\n def normalized_addresses(self,\n addresses=None, quote=True, with_keys=False,\n force_name=False):\n if addresses is None:\n addresses = self\n elif not addresses:\n addresses = []\n def fmt(ai):\n email = ai.address\n if with_keys and ai.keys:\n fp = ai.keys[0].get('fingerprint')\n epart = '<%s%s>' % (email, fp and ('#%s' % fp) or '')\n else:\n epart = '<%s>' % email\n if ai.fn:\n return ' '.join([quote and self.quote(ai.fn) or ai.fn, epart])\n elif force_name:\n return ' '.join([quote and self.quote(email) or email, epart])\n else:\n return epart\n return [fmt(ai) for ai in addresses]\n\n def normalized(self, **kwargs):\n return ', '.join(self.normalized_addresses(**kwargs))\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/emails.py", "content": "# vim: set fileencoding=utf-8 :\n#\n# FIXME: Refactor this monster into mailpile.mailutils.*\n#\nfrom __future__ import print_function\nimport base64\nimport copy\nimport email.header\nimport email.parser\nimport email.utils\nimport errno\nimport mailbox\nimport mimetypes\nimport os\nimport quopri\nimport random\nimport re\nimport StringIO\nimport threading\nimport traceback\nfrom email import encoders\nfrom email.mime.base import MIMEBase\nfrom email.mime.image import MIMEImage\nfrom email.mime.multipart import MIMEMultipart\nfrom email.mime.text import MIMEText\nfrom email.mime.application import MIMEApplication\nfrom mailpile.util import *\nfrom platform import system\nfrom urllib import quote, unquote\nfrom datetime import datetime, timedelta\n\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.crypto.mime import UnwrapMimeCrypto, MessageAsString\nfrom mailpile.crypto.state import EncryptionInfo, SignatureInfo\nfrom mailpile.eventlog import GetThreadEvent\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.vcard import AddressInfo\nfrom mailpile.mailutils import *\nfrom mailpile.mailutils.addresses import AddressHeaderParser\nfrom mailpile.mailutils.generator import Generator\nfrom mailpile.mailutils.html import extract_text_from_html, clean_html\nfrom mailpile.mailutils.headerprint import HeaderPrints\nfrom mailpile.mailutils.safe import safe_decode_hdr\nfrom mailpile.mailutils.vcal import calendar_parse\n\nGLOBAL_CONTENT_ID_LOCK = MboxLock()\nGLOBAL_CONTENT_ID = random.randint(0, 0xfffffff)\n\ndef MakeContentID():\n global GLOBAL_CONTENT_ID\n with GLOBAL_CONTENT_ID_LOCK:\n GLOBAL_CONTENT_ID += 1\n GLOBAL_CONTENT_ID %= 0xfffffff\n return '%x' % GLOBAL_CONTENT_ID\n\n\ndef MakeBoundary():\n return '==%s==' % okay_random(30)\n\n\ndef MakeMessageID():\n # We generate a message-ID which is almost entirely random; we\n # include an element of the local time (give-or-take 36 hours)\n # to further reduce the odds of any collision.\n return '<%s%x@mailpile>' % (\n okay_random(40), time.time() // (3600*48))\n\n\ndef MakeMessageDate(ts=None):\n # Generate valid dates, but add some jitter to the seconds field\n # so we're not trivially leaking our exact time. We also avoid\n # leaking the time zone.\n return email.utils.formatdate(\n timeval=(ts or time.time()) + (random.randint(0, 60) - 30),\n localtime=False)\n\n\nGLOBAL_PARSE_CACHE_LOCK = MboxLock()\nGLOBAL_PARSE_CACHE = []\n\ndef ClearParseCache(cache_id=None, pgpmime=False, full=False):\n global GLOBAL_PARSE_CACHE\n with GLOBAL_PARSE_CACHE_LOCK:\n GPC = GLOBAL_PARSE_CACHE\n for i in range(0, len(GPC)):\n if (full or\n (pgpmime and GPC[i][1]) or\n (cache_id and GPC[i][0] == cache_id)):\n GPC[i] = (None, None, None)\n\n\ndef ParseMessage(fd, cache_id=None, update_cache=False,\n pgpmime='all', config=None, event=None,\n allow_weak_crypto=False):\n global GLOBAL_PARSE_CACHE\n if not GnuPG:\n pgpmime = False\n\n if cache_id is not None and not update_cache:\n with GLOBAL_PARSE_CACHE_LOCK:\n for cid, pm, message in GLOBAL_PARSE_CACHE:\n if cid == cache_id and pm == pgpmime:\n return message\n\n if pgpmime:\n message = ParseMessage(fd, cache_id=cache_id, pgpmime=False,\n config=config)\n if message is None:\n return None\n if cache_id is not None:\n # Caching is enabled, let's not clobber the encrypted version\n # of this message with a fancy decrypted one.\n message = copy.deepcopy(message)\n def MakeGnuPG(*args, **kwargs):\n ev = event or GetThreadEvent()\n if ev and 'event' not in kwargs:\n kwargs['event'] = ev\n return GnuPG(config, *args, **kwargs)\n\n unwrap_attachments = ('all' in pgpmime or 'att' in pgpmime)\n UnwrapMimeCrypto(message,\n protocols={'openpgp': MakeGnuPG},\n unwrap_attachments=unwrap_attachments,\n require_MDC=(not allow_weak_crypto))\n\n else:\n try:\n if not hasattr(fd, 'read'): # Not a file, is it a function?\n fd = fd()\n safe_assert(hasattr(fd, 'read'))\n except (TypeError, AssertionError):\n return None\n\n message = email.parser.Parser().parse(fd)\n msi = message.signature_info = SignatureInfo(bubbly=False)\n mei = message.encryption_info = EncryptionInfo(bubbly=False)\n for part in message.walk():\n part.signature_info = SignatureInfo(parent=msi)\n part.encryption_info = EncryptionInfo(parent=mei)\n\n if cache_id is not None:\n with GLOBAL_PARSE_CACHE_LOCK:\n # Keep 25 items, put new ones at the front\n GLOBAL_PARSE_CACHE[24:] = []\n GLOBAL_PARSE_CACHE[:0] = [(cache_id, pgpmime, message)]\n\n return message\n\n\ndef GetTextPayload(part):\n mimetype = part.get_content_type() or 'text/plain'\n cte = part.get('content-transfer-encoding', '').lower()\n if mimetype[:5] == 'text/' and cte == 'base64':\n # Mailing lists like to mess with text/plain parts, and Majordomo\n # in particular isn't aware of base64 encoding. Compensate!\n payload = part.get_payload(None, False) or ''\n parts = payload.split('\\n--')\n try:\n parts[0] = base64.b64decode(parts[0])\n except TypeError:\n pass\n return '\\n--'.join(parts)\n else:\n return part.get_payload(None, True) or ''\n\n\ndef ExtractEmails(string, strip_keys=True):\n emails = []\n startcrap = re.compile('^[\\'\\\"<(]')\n endcrap = re.compile('[\\'\\\">);]$')\n string = string.replace('<', ' <').replace('(', ' (')\n for w in [sw.strip() for sw in re.compile('[,\\s]+').split(string)]:\n atpos = w.find('@')\n if atpos >= 0:\n while startcrap.search(w):\n w = w[1:]\n while endcrap.search(w):\n w = w[:-1]\n if w.startswith('mailto:'):\n w = w[7:]\n if '?' in w:\n w = w.split('?')[0]\n if strip_keys and '#' in w[atpos:]:\n w = w[:atpos] + w[atpos:].split('#', 1)[0]\n # E-mail addresses are only allowed to contain ASCII\n # characters, so we just strip everything else away.\n emails.append(CleanText(w,\n banned=CleanText.WHITESPACE,\n replace='_').clean)\n return emails\n\n\ndef ExtractEmailAndName(string):\n email = (ExtractEmails(string) or [''])[0]\n name = (string\n .replace(email, '')\n .replace('<>', '')\n .replace('\"', '')\n .replace('(', '')\n .replace(')', '')).strip()\n return email, (name or email)\n\n\ndef CleanHeaders(msg, copy_all=True, tombstones=False):\n clean_headers = []\n address_headers_lower = [h.lower() for h in Email.ADDRESS_HEADERS]\n for key, value in msg.items():\n lkey = key.lower()\n\n # Remove headers we don't want to expose\n if (lkey.startswith('x-mp-internal-') or\n lkey in ('bcc', 'encryption', 'attach-pgp-pubkey')):\n if tombstones:\n clean_headers.append((key, None))\n\n # Strip the #key part off any e-mail addresses:\n elif lkey in address_headers_lower:\n if '#' in value:\n clean_headers.append((key, re.sub(\n r'(@[^<>\\s#]+)#[a-fxA-F0-9]+([>,\\s]|$)', r'\\1\\2', value)))\n elif copy_all:\n clean_headers.append((key, value))\n elif copy_all:\n clean_headers.append((key, value))\n\n return clean_headers\n\n\ndef CleanMessage(config, msg):\n replacements = CleanHeaders(msg, copy_all=False, tombstones=True)\n\n for key, val in replacements:\n del msg[key]\n for key, val in replacements:\n if val:\n msg[key] = val\n\n return msg\n\n\ndef PrepareMessage(config, msg,\n sender=None, rcpts=None, events=None, bounce=False):\n msg = copy.deepcopy(msg)\n\n # Short circuit if this message has already been prepared.\n if ('x-mp-internal-sender' in msg and\n 'x-mp-internal-rcpts' in msg and\n not bounce):\n return (sender or msg['x-mp-internal-sender'],\n rcpts or [r.strip()\n for r in msg['x-mp-internal-rcpts'].split(',')],\n msg,\n events)\n\n crypto_policy = 'default'\n crypto_format = 'default'\n\n rcpts = rcpts or []\n if bounce:\n safe_assert(len(rcpts) > 0)\n\n # Iterate through headers to figure out what we want to do...\n need_rcpts = not rcpts\n for hdr, val in msg.items():\n lhdr = hdr.lower()\n if lhdr == 'from':\n sender = sender or val\n elif lhdr == 'encryption':\n crypto_policy = val\n elif need_rcpts and lhdr in ('to', 'cc', 'bcc'):\n rcpts += AddressHeaderParser(val).addresses_list(with_keys=True)\n\n # Are we sane?\n if not sender:\n raise NoFromAddressError()\n if not rcpts:\n raise NoRecipientError()\n\n # Are we encrypting? Signing?\n crypto_policy = crypto_policy.lower()\n if crypto_policy == 'default':\n crypto_policy = config.prefs.crypto_policy.lower()\n\n sender = AddressHeaderParser(sender)[0].address\n\n # FIXME: Shouldn't this be using config.get_profile instead?\n profile = config.vcards.get_vcard(sender)\n if profile:\n crypto_format = (profile.crypto_format or crypto_format).lower()\n if crypto_format == 'default':\n crypto_format = 'prefer_inline' if config.prefs.inline_pgp else ''\n\n # Extract just the e-mail addresses from the RCPT list, make unique\n rcpts, rr = [], rcpts\n for r in rr:\n for e in AddressHeaderParser(r).addresses_list(with_keys=True):\n if e not in rcpts:\n rcpts.append(e)\n\n # Bouncing disables all transformations, including crypto.\n if not bounce:\n # This is the BCC hack that Brennan hates!\n if config.prefs.always_bcc_self and sender not in rcpts:\n rcpts += [sender]\n\n # Add headers we require\n while 'date' in msg:\n del msg['date']\n msg['Date'] = MakeMessageDate()\n\n import mailpile.plugins\n plugins = mailpile.plugins.PluginManager()\n\n # Perform pluggable content transformations\n sender, rcpts, msg, junk = plugins.outgoing_email_content_transform(\n config, sender, rcpts, msg)\n\n # Perform pluggable encryption transformations\n sender, rcpts, msg, matched = plugins.outgoing_email_crypto_transform(\n config, sender, rcpts, msg,\n crypto_policy=crypto_policy,\n crypto_format=crypto_format,\n cleaner=lambda m: CleanMessage(config, m))\n\n if crypto_policy and (crypto_policy != 'none') and not matched:\n raise ValueError(_('Unknown crypto policy: %s') % crypto_policy)\n\n rcpts = set([r.rsplit('#', 1)[0] for r in rcpts])\n msg['x-mp-internal-readonly'] = str(int(time.time()))\n msg['x-mp-internal-sender'] = sender\n msg['x-mp-internal-rcpts'] = ', '.join(rcpts)\n return (sender, rcpts, msg, events)\n\n\nclass Email(object):\n \"\"\"This is a lazy-loading object representing a single email.\"\"\"\n\n def __init__(self, idx, msg_idx_pos,\n msg_parsed=None, msg_parsed_pgpmime=(None, None),\n msg_info=None, ephemeral_mid=None):\n self.index = idx\n self.config = idx.config\n self.msg_idx_pos = msg_idx_pos\n self.ephemeral_mid = ephemeral_mid\n self.reset_caches(msg_parsed=msg_parsed,\n msg_parsed_pgpmime=msg_parsed_pgpmime,\n msg_info=msg_info,\n clear_parse_cache=False)\n\n def msg_mid(self):\n return self.ephemeral_mid or b36(self.msg_idx_pos)\n\n @classmethod\n def encoded_hdr(self, msg, hdr, value=None):\n hdr_value = value or (msg and msg.get(hdr)) or ''\n try:\n hdr_value.encode('us-ascii')\n except (UnicodeEncodeError, UnicodeDecodeError):\n if hdr.lower() in ('from', 'to', 'cc', 'bcc'):\n addrs = []\n for addr in [a.strip() for a in hdr_value.split(',')]:\n name, part = [], []\n words = addr.split()\n for w in words:\n if w[0] == '<' or '@' in w:\n part.append((w, 'us-ascii'))\n else:\n name.append(w)\n if name:\n name = ' '.join(name)\n try:\n part[0:0] = [(name.encode('us-ascii'), 'us-ascii')]\n except:\n part[0:0] = [(name, 'utf-8')]\n addrs.append(email.header.make_header(part).encode())\n hdr_value = ', '.join(addrs)\n else:\n parts = [(hdr_value, 'utf-8')]\n hdr_value = email.header.make_header(parts).encode()\n return hdr_value\n\n @classmethod\n def Create(cls, idx, mbox_id, mbx,\n msg_to=None, msg_cc=None, msg_bcc=None, msg_from=None,\n msg_subject=None, msg_text='', msg_references=None,\n msg_id=None, msg_atts=None, msg_headers=None,\n save=True, ephemeral_mid='not-saved', append_sig=True,\n use_default_from=True):\n msg = MIMEMultipart(boundary=MakeBoundary())\n msg.signature_info = msi = SignatureInfo(bubbly=False)\n msg.encryption_info = mei = EncryptionInfo(bubbly=False)\n msg_ts = int(time.time())\n\n if msg_from:\n from_email = AddressHeaderParser(unicode_data=msg_from)[0].address\n from_profile = idx.config.get_profile(email=from_email)\n elif use_default_from:\n from_profile = idx.config.get_profile()\n from_email = from_profile.get('email', None)\n from_name = from_profile.get('name', None)\n if from_email and from_name:\n msg_from = '%s <%s>' % (from_name, from_email)\n else:\n from_email = from_profile = from_name = None\n\n if msg_from:\n msg['From'] = cls.encoded_hdr(None, 'from', value=msg_from)\n\n msg['Date'] = MakeMessageDate(msg_ts)\n msg['Message-Id'] = msg_id or MakeMessageID()\n msg_subj = (msg_subject or '')\n msg['Subject'] = cls.encoded_hdr(None, 'subject', value=msg_subj)\n\n # Privacy trade-off: we want to help recipients do profiling and\n # discard poorly forged messages that are not from from Mailpile.\n # However, we don't want to leak too many details for privacy and\n # security reasons. So no: version or platform info, just the word\n # Mailpile. This will probably be obvious to a truly hostile\n # adversary anyway from other details.\n msg['User-Agent'] = 'Mailpile'\n\n ahp = AddressHeaderParser()\n norm = lambda a: ', '.join(sorted(list(set(ahp.normalized_addresses(\n addresses=a, with_keys=True, force_name=True)))))\n if msg_to:\n msg['To'] = cls.encoded_hdr(None, 'to', value=norm(msg_to))\n if msg_cc:\n msg['Cc'] = cls.encoded_hdr(None, 'cc', value=norm(msg_cc))\n if msg_bcc:\n msg['Bcc'] = cls.encoded_hdr(None, 'bcc', value=norm(msg_bcc))\n if msg_references:\n msg['In-Reply-To'] = msg_references[-1]\n msg['References'] = ', '.join(msg_references)\n\n if msg_text:\n try:\n msg_text.encode('us-ascii')\n charset = 'us-ascii'\n except (UnicodeEncodeError, UnicodeDecodeError):\n charset = 'utf-8'\n tp = MIMEText(msg_text, _subtype='plain', _charset=charset)\n tp.signature_info = SignatureInfo(parent=msi)\n tp.encryption_info = EncryptionInfo(parent=mei)\n msg.attach(tp)\n del tp['MIME-Version']\n\n for k, v in (msg_headers or []):\n msg[k] = v\n\n if msg_atts:\n for att in msg_atts:\n att = copy.deepcopy(att)\n att.signature_info = SignatureInfo(parent=msi)\n att.encryption_info = EncryptionInfo(parent=mei)\n# Disabled for now.\n# if att.get('content-id') is None:\n# att.add_header('Content-Id', MakeContentID())\n msg.attach(att)\n del att['MIME-Version']\n\n # Determine if we want to attach a PGP public key due to policy and\n # timing...\n if (idx.config.prefs.gpg_email_key and\n from_profile and\n 'send_keys' in from_profile.get('crypto_format', 'none')):\n from mailpile.plugins.crypto_policy import CryptoPolicy\n addrs = ExtractEmails(norm(msg_to) + norm(msg_cc) + norm(msg_bcc))\n if CryptoPolicy.ShouldAttachKey(idx.config, emails=addrs):\n msg[\"Attach-PGP-Pubkey\"] = \"Yes\"\n\n if save:\n msg_key = mbx.add(MessageAsString(msg))\n msg_to = msg_cc = []\n msg_ptr = mbx.get_msg_ptr(mbox_id, msg_key)\n msg_id = idx.get_msg_id(msg, msg_ptr)\n msg_idx, msg_info = idx.add_new_msg(msg_ptr, msg_id, msg_ts,\n msg_from, msg_to, msg_cc, 0,\n msg_subj, '', [])\n idx.set_conversation_ids(msg_info[idx.MSG_MID], msg,\n subject_threading=False)\n return cls(idx, msg_idx)\n else:\n msg_info = idx.edit_msg_info(idx.BOGUS_METADATA[:],\n msg_mid=ephemeral_mid or '',\n msg_id=msg['Message-ID'],\n msg_ts=msg_ts,\n msg_subject=msg_subj,\n msg_from=msg_from,\n msg_to=msg_to,\n msg_cc=msg_cc)\n return cls(idx, -1,\n msg_info=msg_info,\n msg_parsed=msg,\n msg_parsed_pgpmime=('basic', msg),\n ephemeral_mid=ephemeral_mid)\n\n def is_editable(self, quick=False):\n if self.ephemeral_mid:\n return True\n if not self.config.is_editable_message(self.get_msg_info()):\n return False\n if quick:\n return True\n return ('x-mp-internal-readonly' not in self.get_msg(pgpmime=False))\n\n MIME_HEADERS = ('mime-version', 'content-type', 'content-disposition',\n 'content-transfer-encoding')\n UNEDITABLE_HEADERS = ('message-id', ) + MIME_HEADERS\n MANDATORY_HEADERS = ('From', 'To', 'Cc', 'Bcc', 'Subject',\n 'Encryption', 'Attach-PGP-Pubkey')\n ADDRESS_HEADERS = ('From', 'To', 'Cc', 'Bcc', 'Reply-To')\n HEADER_ORDER = {\n 'in-reply-to': -2,\n 'references': -1,\n 'date': 1,\n 'from': 2,\n 'subject': 3,\n 'to': 4,\n 'cc': 5,\n 'bcc': 6,\n 'encryption': 98,\n 'attach-pgp-pubkey': 99,\n }\n\n def _attachment_aid(self, att):\n aid = att.get('aid')\n if not aid:\n cid = att.get('content-id') # This comes from afar and might\n # be malicious, so check it.\n if (cid and\n cid == CleanText(cid, banned=(CleanText.WHITESPACE +\n CleanText.FS)).clean):\n aid = cid\n else:\n aid = 'part-%s' % att['count']\n return aid\n\n def get_editing_strings(self, tree=None, build_tree=True):\n if build_tree:\n tree = self.get_message_tree(want=['editing_strings'], tree=tree)\n\n strings = {\n 'from': '', 'to': '', 'cc': '', 'bcc': '', 'subject': '',\n 'encryption': '', 'attach-pgp-pubkey': '', 'attachments': {}\n }\n header_lines = []\n body_lines = []\n\n # We care about header order and such things...\n hdrs = dict([(h.lower(), h) for h in tree['headers'].keys()\n if h.lower() not in self.UNEDITABLE_HEADERS])\n for mandate in self.MANDATORY_HEADERS:\n hdrs[mandate.lower()] = hdrs.get(mandate.lower(), mandate)\n keys = hdrs.keys()\n keys.sort(key=lambda k: (self.HEADER_ORDER.get(k.lower(), 99), k))\n lowman = [m.lower() for m in self.MANDATORY_HEADERS]\n lowadr = [m.lower() for m in self.ADDRESS_HEADERS]\n for hdr in [hdrs[k] for k in keys]:\n data = tree['headers'].get(hdr, '')\n lhdr = hdr.lower()\n if lhdr in lowadr and lhdr in lowman:\n adata = tree.get('addresses', {}).get(lhdr, None)\n if adata is None:\n adata = AddressHeaderParser(data)\n strings[lhdr] = adata.normalized()\n elif lhdr in lowman:\n strings[lhdr] = unicode(data)\n else:\n header_lines.append(unicode('%s: %s' % (hdr, data)))\n\n for att in tree['attachments']:\n aid = self._attachment_aid(att)\n strings['attachments'][aid] = (att['filename'] or '(unnamed)')\n\n if not strings['encryption']:\n strings['encryption'] = unicode(self.config.prefs.crypto_policy)\n\n def _fixup(t):\n try:\n return unicode(t)\n except UnicodeDecodeError:\n return t.decode('utf-8')\n\n strings['headers'] = '\\n'.join(header_lines).replace('\\r\\n', '\\n')\n strings['body'] = unicode(''.join([_fixup(t['data'])\n for t in tree['text_parts']])\n ).replace('\\r\\n', '\\n')\n\n return strings\n\n def get_editing_string(self, tree=None,\n estrings=None,\n attachment_headers=True,\n build_tree=True):\n if estrings is None:\n estrings = self.get_editing_strings(tree=tree,\n build_tree=build_tree)\n\n bits = [estrings['headers']] if estrings['headers'] else []\n for mh in self.MANDATORY_HEADERS:\n bits.append('%s: %s' % (mh, estrings[mh.lower()]))\n\n if attachment_headers:\n for aid in sorted(estrings['attachments'].keys()):\n bits.append('Attachment-%s: %s'\n % (aid, estrings['attachments'][aid]))\n bits.append('')\n bits.append(estrings['body'])\n return '\\n'.join(bits)\n\n def _update_att_name(self, part, filename):\n try:\n del part['Content-Disposition']\n except KeyError:\n pass\n part.add_header('Content-Disposition', 'attachment',\n filename=filename)\n return part\n\n def _make_attachment(self, fn, msg, filedata=None):\n if filedata and fn in filedata:\n data = filedata[fn]\n else:\n if isinstance(fn, unicode):\n fn = fn.encode('utf-8')\n data = open(fn, 'rb').read()\n ctype, encoding = mimetypes.guess_type(fn)\n maintype, subtype = (ctype or 'application/octet-stream').split('/', 1)\n if maintype == 'image':\n att = MIMEImage(data, _subtype=subtype)\n else:\n att = MIMEBase(maintype, subtype)\n att.set_payload(data)\n encoders.encode_base64(att)\n# Disabled for now.\n# att.add_header('Content-Id', MakeContentID())\n\n # FS paths are strings of bytes, should be represented as utf-8 for\n # correct header encoding.\n base_fn = os.path.basename(fn)\n if not isinstance(base_fn, unicode):\n base_fn = base_fn.decode('utf-8')\n\n att.add_header('Content-Disposition', 'attachment',\n filename=self.encoded_hdr(None, 'file', base_fn))\n\n att.signature_info = SignatureInfo(parent=msg.signature_info)\n att.encryption_info = EncryptionInfo(parent=msg.encryption_info)\n return att\n\n def update_from_string(self, session, data, final=False):\n if not self.is_editable():\n raise NotEditableError(_('Message or mailbox is read-only.'))\n\n oldmsg = self.get_msg()\n if not data:\n outmsg = oldmsg\n\n else:\n newmsg = email.parser.Parser().parsestr(data.encode('utf-8'))\n outmsg = MIMEMultipart(boundary=MakeBoundary())\n outmsg.signature_info = SignatureInfo(bubbly=False)\n outmsg.encryption_info = EncryptionInfo(bubbly=False)\n\n # Copy over editable headers from the input string, skipping blanks\n for hdr in newmsg.keys():\n if hdr.startswith('Attachment-') or hdr == 'Attachment':\n pass\n else:\n encoded_hdr = self.encoded_hdr(newmsg, hdr)\n if len(encoded_hdr.strip()) > 0:\n if encoded_hdr == '!KEEP':\n if hdr in oldmsg:\n outmsg[hdr] = oldmsg[hdr]\n else:\n outmsg[hdr] = encoded_hdr\n\n # Copy over the uneditable headers from the old message\n for hdr in oldmsg.keys():\n if ((hdr.lower() not in self.MIME_HEADERS)\n and (hdr.lower() in self.UNEDITABLE_HEADERS)):\n outmsg[hdr] = oldmsg[hdr]\n\n # Copy the message text\n new_body = newmsg.get_payload().decode('utf-8')\n target_width = self.config.prefs.line_length\n if target_width >= 40 and 'x-mp-internal-no-reflow' not in newmsg:\n new_body = reflow_text(new_body, target_width=target_width)\n try:\n new_body.encode('us-ascii')\n charset = 'us-ascii'\n except (UnicodeEncodeError, UnicodeDecodeError):\n charset = 'utf-8'\n\n tp = MIMEText(new_body, _subtype='plain', _charset=charset)\n tp.signature_info = SignatureInfo(parent=outmsg.signature_info)\n tp.encryption_info = EncryptionInfo(parent=outmsg.encryption_info)\n outmsg.attach(tp)\n del tp['MIME-Version']\n\n # FIXME: Use markdown and template to generate fancy HTML part?\n\n # Copy the attachments we are keeping\n attachments = [h for h in newmsg.keys()\n if h.lower().startswith('attachment')]\n if attachments:\n oldtree = self.get_message_tree(want=['attachments'])\n for att in oldtree['attachments']:\n hdr = 'Attachment-%s' % self._attachment_aid(att)\n if hdr in attachments:\n outmsg.attach(self._update_att_name(att['part'],\n newmsg[hdr]))\n attachments.remove(hdr)\n\n # Attach some new files?\n for hdr in attachments:\n try:\n att = self._make_attachment(newmsg[hdr], outmsg)\n outmsg.attach(att)\n del att['MIME-Version']\n except:\n pass # FIXME: Warn user that failed...\n\n # Save result back to mailbox\n if final:\n sender, rcpts, outmsg, ev = PrepareMessage(self.config, outmsg)\n return self.update_from_msg(session, outmsg)\n\n def update_from_msg(self, session, newmsg):\n if not self.is_editable():\n raise NotEditableError(_('Message or mailbox is read-only.'))\n\n if self.ephemeral_mid:\n self.reset_caches(clear_parse_cache=False,\n msg_parsed=newmsg,\n msg_parsed_pgpmime=('basic', newmsg),\n msg_info=self.msg_info)\n\n else:\n mbx, ptr, fd = self.get_mbox_ptr_and_fd()\n fd.close() # Windows needs this\n\n # OK, adding to the mailbox worked\n newptr = ptr[:MBX_ID_LEN] + mbx.add(MessageAsString(newmsg))\n self.update_parse_cache(newmsg)\n\n # Remove the old message...\n mbx.remove_by_ptr(ptr)\n\n # FIXME: We should DELETE the old version from the index first.\n\n # Update the in-memory-index\n mi = self.get_msg_info()\n mi[self.index.MSG_PTRS] = newptr\n self.index.set_msg_at_idx_pos(self.msg_idx_pos, mi)\n self.index.index_email(session, Email(self.index, self.msg_idx_pos))\n self.reset_caches(clear_parse_cache=False)\n\n return self\n\n def reset_caches(self,\n msg_info=None,\n msg_parsed=None, msg_parsed_pgpmime=(None, None),\n clear_parse_cache=True):\n self.msg_info = msg_info\n self.msg_parsed = msg_parsed\n self.msg_parsed_pgpmime = msg_parsed_pgpmime\n if clear_parse_cache:\n self.clear_from_parse_cache()\n\n def update_parse_cache(self, newmsg):\n cache_id = self.get_cache_id()\n if cache_id:\n with GLOBAL_PARSE_CACHE_LOCK:\n GPC = GLOBAL_PARSE_CACHE\n for i in range(0, len(GPC)):\n if GPC[i][0] == cache_id:\n GPC[i] = (cache_id, False, newmsg)\n\n def clear_from_parse_cache(self):\n cache_id = self.get_cache_id()\n if cache_id:\n ClearParseCache(cache_id=cache_id)\n\n def delete_message(self, session, flush=True, keep=0):\n mi = self.get_msg_info()\n removed, failed, mailboxes = [], [], []\n kept = keep\n allow_deletion = session.config.prefs.allow_deletion\n for msg_ptr, mbox, fd in self.index.enumerate_ptrs_mboxes_fds(mi):\n try:\n if mbox:\n try:\n if keep > 0:\n # Note: This will keep messages in the order of\n # preference implemented by enumerate_ptrs_...\n # FIXME: Allow more nuanced behaviour here.\n mbox.get_file_by_ptr(msg_ptr)\n keep -= 1\n elif allow_deletion:\n mbox.remove_by_ptr(msg_ptr)\n else:\n # FIXME: Allow deletion of local copies ONLY\n raise ValueError(\"Deletion is forbidden\")\n except (KeyError, IndexError):\n # Already gone!\n pass\n mailboxes.append(mbox)\n removed.append(msg_ptr)\n except (IOError, OSError, ValueError, AttributeError) as e:\n failed.append(msg_ptr)\n print('FIXME: Could not delete %s: %s' % (msg_ptr, e))\n\n if allow_deletion and not failed and not kept:\n self.index.delete_msg_at_idx_pos(session, self.msg_idx_pos,\n keep_msgid=False)\n if flush:\n for m in mailboxes:\n m.flush()\n return (not failed, [])\n else:\n return (not failed, mailboxes)\n\n def get_msg_info(self, field=None, uncached=False):\n if (uncached or not self.msg_info) and not self.ephemeral_mid:\n self.msg_info = self.index.get_msg_at_idx_pos(self.msg_idx_pos)\n if field is None:\n return self.msg_info\n else:\n return self.msg_info[field]\n\n def get_mbox_ptr_and_fd(self):\n mi = self.get_msg_info()\n for msg_ptr, mbox, fd in self.index.enumerate_ptrs_mboxes_fds(mi):\n if fd is not None:\n # FIXME: How do we know we have the right message?\n return mbox, msg_ptr, FixupForWith(fd)\n return None, None, None\n\n def get_file(self):\n return self.get_mbox_ptr_and_fd()[2]\n\n def get_msg_size(self):\n mbox, ptr, fd = self.get_mbox_ptr_and_fd()\n with fd:\n fd.seek(0, 2)\n return fd.tell()\n\n def get_metadata_kws(self):\n # FIXME: Track these somehow...\n return []\n\n def get_cache_id(self):\n if (self.msg_idx_pos >= 0) and not self.ephemeral_mid:\n return '%s/%s' % (self.index, self.msg_idx_pos)\n else:\n return None\n\n def _get_parsed_msg(self, pgpmime, update_cache=False):\n weak_crypto_max_age = self.config.prefs.weak_crypto_max_age\n allow_weak_crypto = False\n if weak_crypto_max_age > 0:\n ts = int(self.get_msg_info(self.index.MSG_DATE) or '0', 36)\n allow_weak_crypto = (ts < weak_crypto_max_age)\n return ParseMessage(self.get_file,\n cache_id=self.get_cache_id(),\n update_cache=update_cache,\n pgpmime=pgpmime,\n config=self.config,\n allow_weak_crypto=allow_weak_crypto,\n event=GetThreadEvent())\n\n def _update_crypto_state(self):\n if not (self.config.tags and\n self.msg_idx_pos >= 0 and\n self.msg_parsed_pgpmime[0] and\n self.msg_parsed_pgpmime[1] and\n not self.ephemeral_mid):\n return\n\n import mailpile.plugins.cryptostate as cs\n kw = cs.meta_kw_extractor(self.index,\n self.msg_mid(),\n self.msg_parsed_pgpmime[1],\n 0, 0) # msg_size, msg_ts\n\n # We do NOT want to update tags if we are getting back\n # a none/none state, as that can happen for the more\n # complex nested crypto-in-text messages, which a more\n # forceful parse of the message may have caught earlier.\n no_sig = self.config.get_tag('mp_sig-none')\n no_sig = no_sig and '%s:in' % no_sig._key\n no_enc = self.config.get_tag('mp_enc-none')\n no_enc = no_enc and '%s:in' % no_enc._key\n if no_sig not in kw or no_enc not in kw:\n msg_info = self.get_msg_info()\n msg_tags = msg_info[self.index.MSG_TAGS].split(',')\n msg_tags = sorted([t for t in msg_tags if t])\n\n # Note: this has the side effect of cleaning junk off\n # the tag list, not just updating crypto state.\n def tcheck(tag_id):\n tag = self.config.get_tag(tag_id)\n return (tag and tag.slug[:6] not in ('mp_enc', 'mp_sig'))\n new_tags = sorted([t for t in msg_tags if tcheck(t)] +\n [ti.split(':', 1)[0] for ti in kw\n if ti.endswith(':in')])\n\n if msg_tags != new_tags:\n msg_info[self.index.MSG_TAGS] = ','.join(new_tags)\n self.index.set_msg_at_idx_pos(self.msg_idx_pos, msg_info)\n\n def get_msg(self, pgpmime='default', crypto_state_feedback=True):\n if pgpmime:\n if pgpmime == 'default':\n pgpmime = 'basic' if self.is_editable() else 'all'\n\n if self.msg_parsed_pgpmime[0] == pgpmime:\n result = self.msg_parsed_pgpmime[1]\n else:\n result = self._get_parsed_msg(pgpmime)\n self.msg_parsed_pgpmime = (pgpmime, result)\n\n # Post-parse, we want to make sure that the crypto-state\n # recorded on this message's metadata is up to date.\n if crypto_state_feedback:\n self._update_crypto_state()\n else:\n if not self.msg_parsed:\n self.msg_parsed = self._get_parsed_msg(pgpmime)\n result = self.msg_parsed\n if not result:\n raise IndexError(_('Message not found'))\n return result\n\n def is_thread(self):\n return ((self.get_msg_info(self.index.MSG_THREAD_MID)) or\n (0 < len(self.get_msg_info(self.index.MSG_REPLIES))))\n\n def get(self, field, default=''):\n \"\"\"Get one (or all) indexed fields for this mail.\"\"\"\n field = field.lower()\n if field == 'subject':\n return self.get_msg_info(self.index.MSG_SUBJECT)\n elif field == 'from':\n return self.get_msg_info(self.index.MSG_FROM)\n else:\n raw = ' '.join(self.get_msg(pgpmime=False).get_all(field, default))\n return safe_decode_hdr(hdr=raw) or raw\n\n def get_sender(self):\n try:\n ahp = AddressHeaderParser(unicode_data=self.get('from'))\n return ahp[0].address\n except IndexError:\n return None\n\n def get_headerprints(self):\n return HeaderPrints(self.get_msg(pgpmime='basic'))\n\n def get_msg_summary(self):\n # We do this first to make sure self.msg_info is loaded\n msg_mid = self.get_msg_info(self.index.MSG_MID)\n return [\n msg_mid,\n self.get_msg_info(self.index.MSG_ID),\n self.get_msg_info(self.index.MSG_FROM),\n self.index.expand_to_list(self.msg_info),\n self.get_msg_info(self.index.MSG_SUBJECT),\n self.get_msg_info(self.index.MSG_BODY),\n self.get_msg_info(self.index.MSG_DATE),\n self.get_msg_info(self.index.MSG_TAGS).split(','),\n self.is_editable(quick=True)\n ]\n\n def _find_attachments(self, att_id, negative=False):\n msg = self.get_msg()\n count = 0\n for part in (msg.walk() if msg else []):\n mimetype = (part.get_content_type() or 'text/plain').lower()\n if mimetype.startswith('multipart/'):\n continue\n\n count += 1\n content_id = part.get('content-id', '')\n pfn = safe_decode_hdr(hdr=part.get_filename() or '')\n\n if (('*' == att_id)\n or ('#%s' % count == att_id)\n or ('part-%s' % count == att_id)\n or (content_id == att_id)\n or (mimetype == att_id)\n or (pfn.lower().endswith('.%s' % att_id))\n or (pfn == att_id)):\n if not negative:\n yield (count, content_id, pfn, mimetype, part)\n elif negative:\n yield (count, content_id, pfn, mimetype, part)\n\n def add_attachments(self, session, filenames, filedata=None):\n if not self.is_editable():\n raise NotEditableError(_('Message or mailbox is read-only.'))\n msg = self.get_msg()\n for fn in filenames:\n att = self._make_attachment(fn, msg, filedata=filedata)\n msg.attach(att)\n del att['MIME-Version']\n return self.update_from_msg(session, msg)\n\n def remove_attachments(self, session, *att_ids):\n if not self.is_editable():\n raise NotEditableError(_('Message or mailbox is read-only.'))\n\n remove = []\n for att_id in att_ids:\n for count, cid, pfn, mt, part in self._find_attachments(att_id):\n remove.append(self._attachment_aid({\n 'msg_mid': self.msg_mid(),\n 'count': count,\n 'content-id': cid,\n 'filename': pfn,\n }))\n\n es = self.get_editing_strings()\n es['headers'] = None\n for k in remove:\n if k in es['attachments']:\n del es['attachments'][k]\n\n estring = self.get_editing_string(estrings=es)\n return self.update_from_string(session, estring)\n\n def extract_attachment(self, session, att_id, name_fmt=None, mode='get'):\n extracted = 0\n filename, attributes = '', {}\n for (count, content_id, pfn, mimetype, part\n ) in self._find_attachments(att_id):\n payload = part.get_payload(None, True) or ''\n attributes = {\n 'msg_mid': self.msg_mid(),\n 'count': count,\n 'length': len(payload),\n 'content-id': content_id,\n 'filename': pfn}\n attributes['aid'] = self._attachment_aid(attributes)\n if pfn:\n if '.' in pfn:\n pfn, attributes['att_ext'] = pfn.rsplit('.', 1)\n attributes['att_ext'] = attributes['att_ext'].lower()\n attributes['att_name'] = pfn\n if mimetype:\n attributes['mimetype'] = mimetype\n\n filesize = len(payload)\n if mode.startswith('inline'):\n attributes['data'] = payload\n session.ui.notify(_('Extracted attachment %s') % att_id)\n elif mode.startswith('preview'):\n attributes['thumb'] = True\n attributes['mimetype'] = 'image/jpeg'\n attributes['disposition'] = 'inline'\n thumb = StringIO.StringIO()\n if thumbnail(payload, thumb, height=250):\n attributes['length'] = thumb.tell()\n filename, fd = session.ui.open_for_data(\n name_fmt=name_fmt, attributes=attributes)\n thumb.seek(0)\n fd.write(thumb.read())\n fd.close()\n session.ui.notify(_('Wrote preview to: %s') % filename)\n else:\n session.ui.notify(_('Failed to generate thumbnail'))\n raise UrlRedirectException('/static/img/image-default.png')\n else:\n WHITELIST = ('image/png',\n 'image/gif',\n 'image/jpeg',\n 'image/tiff',\n 'audio/mp3',\n 'audio/ogg',\n 'audio/x-wav',\n 'audio/mpeg',\n 'video/mpeg',\n 'video/ogg',\n 'application/pdf')\n if mode.startswith('get') and mimetype in WHITELIST:\n # This allows the browser to (optionally) handle the\n # content, instead of always forcing a download dialog.\n attributes['disposition'] = 'inline'\n filename, fd = session.ui.open_for_data(\n name_fmt=name_fmt, attributes=attributes)\n fd.write(payload)\n session.ui.notify(_('Wrote attachment to: %s') % filename)\n fd.close()\n extracted += 1\n\n if 0 == extracted:\n session.ui.notify(_('No attachments found for: %s') % att_id)\n return None, None\n else:\n return filename, attributes\n\n def get_message_tags(self):\n tids = self.get_msg_info(self.index.MSG_TAGS).split(',')\n return [self.config.get_tag(t) for t in tids]\n\n def get_message_tree(self, want=None, tree=None, pgpmime='default'):\n msg = self.get_msg(pgpmime=pgpmime)\n want = list(want) if (want is not None) else None\n tree = tree or {'_cleaned': []}\n tree['id'] = self.get_msg_info(self.index.MSG_ID)\n\n if want is not None:\n if 'editing_strings' in want or 'editing_string' in want:\n want.extend(['text_parts', 'headers', 'attachments',\n 'addresses'])\n\n for p in 'text_parts', 'html_parts', 'vcal_parts', 'attachments':\n if want is None or p in want:\n tree[p] = []\n\n if (want is None or 'summary' in want) and 'summary' not in tree:\n tree['summary'] = self.get_msg_summary()\n\n if (want is None or 'tags' in want) and 'tags' not in tree:\n tree['tags'] = self.get_msg_info(self.index.MSG_TAGS).split(',')\n\n if (want is None or 'conversation' in want\n ) and 'conversation' not in tree:\n tree['conversation'] = {}\n conv_id = self.get_msg_info(self.index.MSG_THREAD_MID)\n if conv_id:\n conv_id = conv_id.split('/')[0]\n conv = Email(self.index, int(conv_id, 36))\n tree['conversation'] = convs = [conv.get_msg_summary()]\n for rid in conv.get_msg_info(self.index.MSG_REPLIES\n ).split(','):\n if rid:\n convs.append(Email(self.index, int(rid, 36)\n ).get_msg_summary())\n\n if (want is None or 'headerprints' in want):\n tree['headerprints'] = self.get_headerprints()\n\n if (want is None or 'headers' in want) and 'headers' not in tree:\n tree['headers'] = {}\n for hdr in msg.keys():\n tree['headers'][hdr] = safe_decode_hdr(msg, hdr)\n\n if (want is None or 'headers_lc' in want\n ) and 'headers_lc' not in tree:\n tree['headers_lc'] = {}\n for hdr in msg.keys():\n tree['headers_lc'][hdr.lower()] = safe_decode_hdr(msg, hdr)\n\n if (want is None or 'header_list' in want\n ) and 'header_list' not in tree:\n tree['header_list'] = [(k, safe_decode_hdr(msg, k, hdr=v))\n for k, v in msg.items()]\n\n if (want is None or 'addresses' in want\n ) and 'addresses' not in tree:\n address_headers_lower = [h.lower() for h in self.ADDRESS_HEADERS]\n tree['addresses'] = {}\n for hdr in msg.keys():\n hdrl = hdr.lower()\n if hdrl in address_headers_lower:\n tree['addresses'][hdrl] = AddressHeaderParser(msg[hdr])\n\n # Note: count algorithm must match that used in extract_attachment\n # above\n count = 0\n broken_text_part = None\n for part in msg.walk():\n crypto = {\n 'signature': part.signature_info,\n 'encryption': part.encryption_info}\n\n mimetype = (part.get_content_type() or 'text/plain').lower()\n if (mimetype.startswith('multipart/')\n or mimetype == \"application/pgp-encrypted\"):\n continue\n try:\n if (mimetype == \"application/octet-stream\"\n and part.cryptedcontainer is True):\n continue\n except:\n pass\n\n count += 1\n disposition = part.get('content-disposition', 'inline').lower()\n if (disposition[:6] == 'inline'\n and mimetype.startswith('text/')):\n payload, charset = self.decode_payload(part)\n start = payload[:100].strip()\n\n if mimetype == 'text/html':\n if want is None or 'html_parts' in want:\n tree['html_parts'].append({\n 'charset': charset,\n 'type': 'html',\n 'data': clean_html(payload),\n 'count': count,\n 'mimetype': mimetype,\n 'aid': 'part-%d' % count})\n\n elif mimetype == \"text/calendar\":\n if want is None or 'vcal_parts' in want:\n tree[\"vcal_parts\"].extend(calendar_parse(payload))\n\n elif want is None or 'text_parts' in want:\n for ht in ('', '

2\n and '@' in clines[-2] and '' == clines[-1].strip()):\n current['data'] = ''.join(clines[:-2])\n clines = clines[-2:]\n else:\n clines = []\n\n current = {\n 'type': ltype,\n 'data': ''.join(clines),\n 'charset': charset,\n 'crypto': {\n 'signature': SignatureInfo(parent=psi),\n 'encryption': EncryptionInfo(parent=pei)}}\n parse.append(current)\n if len(parse) == 1 and count and mimetype:\n current['aid'] = 'part-%d' % count\n current['mimetype'] = mimetype\n current['data'] += line\n clines.append(line)\n return parse\n\n BARE_QUOTE_STARTS = re.compile('(?i)^-+\\s*Original Message.*-+$')\n GIT_DIFF_STARTS = re.compile('^diff --git a/.*b/')\n GIT_DIFF_LINE = re.compile('^([ +@-]|index |$)')\n\n def parse_line_type(self, line, block, line_count):\n # FIXME: Detect forwarded messages, ...\n\n if (block in ('body', 'quote', 'barequote')\n and line in ('-- \\n', '-- \\r\\n', '- --\\n', '- --\\r\\n')):\n return 'signature', 'signature'\n\n if block == 'signature':\n return block, block\n\n if block == 'barequote':\n return 'barequote', 'quote'\n\n stripped = line.rstrip()\n\n if stripped == GnuPG.ARMOR_BEGIN_SIGNED:\n return 'pgpbeginsigned', 'pgpbeginsigned'\n if block == 'pgpbeginsigned':\n if line.startswith('Hash: ') or stripped == '':\n return 'pgpbeginsigned', 'pgpbeginsigned'\n else:\n return 'pgpsignedtext', 'pgpsignedtext'\n if block == 'pgpsignedtext':\n if stripped == GnuPG.ARMOR_BEGIN_SIGNATURE:\n return 'pgpsignature', 'pgpsignature'\n else:\n return 'pgpsignedtext', 'pgpsignedtext'\n if block == 'pgpsignature':\n if stripped == GnuPG.ARMOR_END_SIGNATURE:\n return 'pgpend', 'pgpsignature'\n else:\n return 'pgpsignature', 'pgpsignature'\n\n if (stripped == GnuPG.ARMOR_BEGIN_ENCRYPTED\n # This is an EFail mitigation: do not decrypt content\n # inlined somewhere well below a bunch of other stuff.\n # The encrypted content must be high up enough that\n # the user will plausibly see it when reading.\n and line_count < 10 and block == 'body'):\n return 'pgpbegin', 'pgpbegin'\n if block == 'pgpbegin':\n if ':' in line or stripped == '':\n return 'pgpbegin', 'pgpbegin'\n else:\n return 'pgptext', 'pgptext'\n if block == 'pgptext':\n if stripped == GnuPG.ARMOR_END_ENCRYPTED:\n return 'pgpend', 'pgpend'\n else:\n return 'pgptext', 'pgptext'\n\n if self.BARE_QUOTE_STARTS.match(stripped):\n return 'barequote', 'quote'\n\n if block == 'quote':\n if stripped == '':\n return 'quote', 'quote'\n if line.startswith('>'):\n return 'quote', 'quote'\n\n if self.GIT_DIFF_STARTS.match(stripped):\n return 'gitdiff', 'quote'\n\n if block == 'gitdiff':\n if self.GIT_DIFF_LINE.match(stripped):\n return 'gitdiff', 'quote'\n\n return 'body', 'text'\n\n WANT_MSG_TREE_PGP = ('text_parts', 'crypto')\n PGP_OK = {\n 'pgpbeginsigned': 'pgpbeginverified',\n 'pgpsignedtext': 'pgpverifiedtext',\n 'pgpsignature': 'pgpverification',\n 'pgpbegin': 'pgpbeginverified',\n 'pgptext': 'pgpsecuretext',\n 'pgpend': 'pgpverification',\n }\n\n def evaluate_pgp(self, tree, check_sigs=True, decrypt=False,\n crypto_state_feedback=True, event=None):\n if 'text_parts' not in tree:\n return tree\n\n pgpdata = []\n for part in tree['text_parts']:\n if 'crypto' not in part:\n part['crypto'] = {}\n\n ei = si = None\n if check_sigs:\n if part['type'] == 'pgpbeginsigned':\n pgpdata = [part]\n elif part['type'] == 'pgpsignedtext':\n pgpdata.append(part)\n elif part['type'] == 'pgpsignature':\n pgpdata.append(part)\n try:\n gpg = GnuPG(self.config, event=event)\n message = ''.join([p['data'].encode(p['charset'])\n for p in pgpdata])\n si = gpg.verify(message)\n pgpdata[0]['data'] = ''\n pgpdata[1]['crypto']['signature'] = si\n pgpdata[2]['data'] = ''\n\n except Exception as e:\n print(e)\n\n if decrypt:\n if part['type'] in ('pgpbegin', 'pgptext'):\n pgpdata.append(part)\n elif part['type'] == 'pgpend':\n pgpdata.append(part)\n\n data = ''.join([p['data'] for p in pgpdata])\n gpg = GnuPG(self.config, event=event)\n si, ei, text = gpg.decrypt(data)\n\n # FIXME: If the data is binary, we should provide some\n # sort of download link or maybe leave the PGP\n # blob entirely intact, undecoded.\n text, charset = self.decode_text(text, binary=False)\n\n pgpdata[1]['crypto']['encryption'] = ei\n pgpdata[1]['crypto']['signature'] = si\n if ei[\"status\"] == \"decrypted\":\n pgpdata[0]['data'] = \"\"\n pgpdata[1]['data'] = text\n pgpdata[2]['data'] = \"\"\n\n # Bubbling up!\n if (si or ei) and 'crypto' not in tree:\n tree['crypto'] = {'signature': SignatureInfo(bubbly=False),\n 'encryption': EncryptionInfo(bubbly=False)}\n if si:\n si.bubble_up(tree['crypto']['signature'])\n if ei:\n ei.bubble_up(tree['crypto']['encryption'])\n\n # Cleanup, remove empty 'crypto': {} blocks.\n for part in tree['text_parts']:\n if not part['crypto']:\n del part['crypto']\n\n tree['crypto']['signature'].mix_bubbles()\n tree['crypto']['encryption'].mix_bubbles()\n if crypto_state_feedback:\n self._update_crypto_state()\n\n return tree\n\n def _decode_gpg(self, message, decrypted):\n header, body = message.replace('\\r\\n', '\\n').split('\\n\\n', 1)\n for line in header.lower().split('\\n'):\n if line.startswith('charset:'):\n return decrypted.decode(line.split()[1])\n return decrypted.decode('utf-8')\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/generator.py", "content": "# Copyright (C) 2001-2010 Python Software Foundation\n# Contact: email-sig@python.org\n#\n# Updated/forked January 2014 by Bjarni R. Einarsson \n# to match the python 3.x email.generator CRLF control API (linesep=...).\n#\n# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2\n# --------------------------------------------\n#\n# 1. This LICENSE AGREEMENT is between the Python Software Foundation\n# (\"PSF\"), and the Individual or Organization (\"Licensee\") accessing and\n# otherwise using this software (\"Python\") in source or binary form and\n# its associated documentation.\n#\n# 2. Subject to the terms and conditions of this License Agreement, PSF\n# hereby grants Licensee a nonexclusive, royalty-free, world-wide\n# license to reproduce, analyze, test, perform and/or display publicly,\n# prepare derivative works, distribute, and otherwise use Python\n# alone or in any derivative version, provided, however, that PSF's\n# License Agreement and PSF's notice of copyright, i.e., \"Copyright (c)\n# 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights\n# Reserved\" are retained in Python alone or in any derivative version\n# prepared by Licensee.\n#\n# 3. In the event Licensee prepares a derivative work that is based on\n# or incorporates Python or any part thereof, and wants to make\n# the derivative work available to others as provided herein, then\n# Licensee hereby agrees to include in any such work a brief summary of\n# the changes made to Python.\n#\n# 4. PSF is making Python available to Licensee on an \"AS IS\"\n# basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR\n# IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND\n# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS\n# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT\n# INFRINGE ANY THIRD PARTY RIGHTS.\n#\n# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON\n# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS\n# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,\n# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.\n#\n# 6. This License Agreement will automatically terminate upon a material\n# breach of its terms and conditions.\n#\n# 7. Nothing in this License Agreement shall be deemed to create any\n# relationship of agency, partnership, or joint venture between PSF and\n# Licensee. This License Agreement does not grant permission to use PSF\n# trademarks or trade name in a trademark sense to endorse or promote\n# products or services of Licensee, or any third party.\n#\n# 8. By copying, installing or otherwise using Python, Licensee\n# agrees to be bound by the terms and conditions of this License\n# Agreement.\n\n\n\"\"\"Classes to generate plain text from a message object tree.\"\"\"\nfrom __future__ import print_function\n\n__all__ = ['Generator', 'DecodedGenerator']\n\nimport re\nimport sys\nimport time\nimport random\nimport warnings\n\nfrom cStringIO import StringIO\nfrom email.header import Header\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\nUNDERSCORE = '_'\nNL = '\\n'\n\nfcre = re.compile(r'^From ', re.MULTILINE)\n\n\ndef _is8bitstring(s):\n if isinstance(s, str):\n try:\n unicode(s, 'us-ascii')\n except UnicodeError:\n return True\n return False\n\n\nclass Generator:\n \"\"\"Generates output from a Message object tree.\n\n This basic generator writes the message to the given file object as plain\n text.\n \"\"\"\n #\n # Public interface\n #\n\n def __init__(self, outfp,\n mangle_from_=True, maxheaderlen=78, linesep=None):\n \"\"\"Create the generator for message flattening.\n\n outfp is the output file-like object for writing the message to. It\n must have a write() method.\n\n Optional mangle_from_ is a flag that, when True (the default), escapes\n From_ lines in the body of the message by putting a `>' in front of\n them.\n\n Optional maxheaderlen specifies the longest length for a non-continued\n header. When a header line is longer (in characters, with tabs\n expanded to 8 spaces) than maxheaderlen, the header will split as\n defined in the Header class. Set maxheaderlen to zero to disable\n header wrapping. The default is 78, as recommended (but not required)\n by RFC 2822.\n \"\"\"\n self._fp = outfp\n self._mangle_from_ = mangle_from_\n self._maxheaderlen = maxheaderlen\n self._NL = linesep or NL\n\n def write(self, s):\n # Just delegate to the file object\n self._fp.write(s)\n\n def flatten(self, msg, unixfrom=False, linesep=None):\n \"\"\"Print the message object tree rooted at msg to the output file\n specified when the Generator instance was created.\n\n unixfrom is a flag that forces the printing of a Unix From_ delimiter\n before the first object in the message tree. If the original message\n has no From_ delimiter, a `standard' one is crafted. By default, this\n is False to inhibit the printing of any From_ delimiter.\n\n linesep specifies the characters used to indicate a new line in the\n output. The default value is LF (not the standard CRLF).\n\n Note that for subobjects, no From_ line is printed.\n \"\"\"\n if linesep:\n self._NL = linesep\n if unixfrom:\n ufrom = msg.get_unixfrom()\n if not ufrom:\n ufrom = 'From nobody ' + time.ctime(time.time())\n print(ufrom + self._NL, end='', file=self._fp)\n self._write(msg)\n\n def clone(self, fp):\n \"\"\"Clone this generator with the exact same options.\"\"\"\n return self.__class__(fp, self._mangle_from_, self._maxheaderlen)\n\n #\n # Protected interface - undocumented ;/\n #\n\n def _write(self, msg):\n # We can't write the headers yet because of the following scenario:\n # say a multipart message includes the boundary string somewhere in\n # its body. We'd have to calculate the new boundary /before/ we write\n # the headers so that we can write the correct Content-Type:\n # parameter.\n #\n # The way we do this, so as to make the _handle_*() methods simpler,\n # is to cache any subpart writes into a StringIO. The we write the\n # headers and the StringIO contents. That way, subpart handlers can\n # Do The Right Thing, and can still modify the Content-Type: header if\n # necessary.\n oldfp = self._fp\n try:\n self._fp = sfp = StringIO()\n self._dispatch(msg)\n finally:\n self._fp = oldfp\n # Write the headers. First we see if the message object wants to\n # handle that itself. If not, we'll do it generically.\n meth = getattr(msg, '_write_headers', None)\n if meth is None:\n self._write_headers(msg)\n else:\n meth(self)\n self._fp.write(sfp.getvalue())\n\n def _dispatch(self, msg):\n # Get the Content-Type: for the message, then try to dispatch to\n # self._handle__(). If there's no handler for the\n # full MIME type, then dispatch to self._handle_(). If\n # that's missing too, then dispatch to self._writeBody().\n main = msg.get_content_maintype()\n sub = msg.get_content_subtype()\n specific = UNDERSCORE.join((main, sub)).replace('-', '_')\n meth = getattr(self, '_handle_' + specific, None)\n if meth is None:\n generic = main.replace('-', '_')\n meth = getattr(self, '_handle_' + generic, None)\n if meth is None:\n meth = self._writeBody\n meth(msg)\n\n #\n # Default handlers\n #\n\n def _write_headers(self, msg):\n for h, v in msg.items():\n print('%s:' % h, end=' ', file=self._fp)\n if self._maxheaderlen == 0:\n # Explicit no-wrapping\n print(v + self._NL, end='', file=self._fp)\n elif isinstance(v, Header):\n # Header instances know what to do\n hdr = v.encode().replace('\\n', self._NL)\n print(hdr + self._NL, end='', file=self._fp)\n elif _is8bitstring(v):\n # If we have raw 8bit data in a byte string, we have no idea\n # what the encoding is. There is no safe way to split this\n # string. If it's ascii-subset, then we could do a normal\n # ascii split, but if it's multibyte then we could break the\n # string. There's no way to know so the least harm seems to\n # be to not split the string and risk it being too long.\n print(v + self._NL, end='', file=self._fp)\n else:\n # Header's got lots of smarts, so use it. Note that this is\n # fundamentally broken though because we lose idempotency when\n # the header string is continued with tabs. It will now be\n # continued with spaces. This was reversedly broken before we\n # fixed bug 1974. Either way, we lose.\n hdr = Header(v, maxlinelen=self._maxheaderlen, header_name=h\n ).encode().replace('\\n', self._NL)\n print(hdr + self._NL, end='', file=self._fp)\n # A blank line always separates headers from body\n print(self._NL, end='', file=self._fp)\n\n #\n # Handlers for writing types and subtypes\n #\n\n def _handle_text(self, msg):\n payload = msg.get_payload()\n if payload is None:\n return\n if not isinstance(payload, basestring):\n raise TypeError('string payload expected: %s' % type(payload))\n if self._mangle_from_:\n payload = fcre.sub('>From ', payload)\n self._fp.write(payload)\n\n # Default body handler\n _writeBody = _handle_text\n\n def _handle_multipart(self, msg):\n # The trick here is to write out each part separately, merge them all\n # together, and then make sure that the boundary we've chosen isn't\n # present in the payload.\n msgtexts = []\n subparts = msg.get_payload()\n if subparts is None:\n subparts = []\n elif isinstance(subparts, basestring):\n # e.g. a non-strict parse of a message with no starting boundary.\n self._fp.write(subparts)\n return\n elif not isinstance(subparts, list):\n # Scalar payload\n subparts = [subparts]\n for part in subparts:\n s = StringIO()\n g = self.clone(s)\n g.flatten(part, unixfrom=False, linesep=self._NL)\n msgtexts.append(s.getvalue())\n # BAW: What about boundaries that are wrapped in double-quotes?\n boundary = msg.get_boundary()\n if not boundary:\n # Create a boundary that doesn't appear in any of the\n # message texts.\n alltext = self._NL.join(msgtexts)\n boundary = _make_boundary(alltext)\n msg.set_boundary(boundary)\n # If there's a preamble, write it out, with a trailing CRLF\n if msg.preamble is not None:\n if self._mangle_from_:\n preamble = fcre.sub('>From ', msg.preamble)\n else:\n preamble = msg.preamble\n print(preamble + self._NL, end='', file=self._fp)\n # dash-boundary transport-padding CRLF\n print('--' + boundary + self._NL, end='', file=self._fp)\n # body-part\n if msgtexts:\n self._fp.write(msgtexts.pop(0))\n # *encapsulation\n # --> delimiter transport-padding\n # --> CRLF body-part\n for body_part in msgtexts:\n # delimiter transport-padding CRLF\n print(self._NL + '--' + boundary + self._NL, end='', file=self._fp)\n # body-part\n self._fp.write(body_part)\n # close-delimiter transport-padding\n self._fp.write(self._NL + '--' + boundary + '--')\n if msg.epilogue is not None:\n print(self._NL, end='', file=self._fp)\n if self._mangle_from_:\n epilogue = fcre.sub('>From ', msg.epilogue)\n else:\n epilogue = msg.epilogue\n self._fp.write(epilogue)\n\n def _handle_multipart_signed(self, msg):\n # The contents of signed parts has to stay unmodified in order to keep\n # the signature intact per RFC1847 2.1, so we disable header wrapping.\n # RDM: This isn't enough to completely preserve the part, but it helps.\n # BRE: Disabled! We are using this to generate the stuff we sign, so\n # we actually want the logic UNCHANGED.\n return self._handle_multipart(msg)\n # Disabled the following...\n old_maxheaderlen = self._maxheaderlen\n try:\n self._maxheaderlen = 0\n self._handle_multipart(msg)\n finally:\n self._maxheaderlen = old_maxheaderlen\n\n def _handle_message_delivery_status(self, msg):\n # We can't just write the headers directly to self's file object\n # because this will leave an extra newline between the last header\n # block and the boundary. Sigh.\n blocks = []\n for part in msg.get_payload():\n s = StringIO()\n g = self.clone(s)\n g.flatten(part, unixfrom=False, linesep=self._NL)\n text = s.getvalue()\n lines = text.split(self._NL)\n # Strip off the unnecessary trailing empty line\n if lines and lines[-1] == '':\n blocks.append(self._NL.join(lines[:-1]))\n else:\n blocks.append(text)\n # Now join all the blocks with an empty line. This has the lovely\n # effect of separating each block with an empty line, but not adding\n # an extra one after the last one.\n self._fp.write(self._NL.join(blocks))\n\n def _handle_message(self, msg):\n s = StringIO()\n g = self.clone(s)\n # The payload of a message/rfc822 part should be a multipart sequence\n # of length 1. The zeroth element of the list should be the Message\n # object for the subpart. Extract that object, stringify it, and\n # write it out.\n # Except, it turns out, when it's a string instead, which happens when\n # and only when HeaderParser is used on a message of mime type\n # message/rfc822. Such messages are generated by, for example,\n # Groupwise when forwarding unadorned messages. (Issue 7970.) So\n # in that case we just emit the string body.\n payload = msg.get_payload()\n if isinstance(payload, list):\n g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL)\n payload = s.getvalue()\n self._fp.write(payload)\n\n\n_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'\n\n\nclass DecodedGenerator(Generator):\n \"\"\"Generates a text representation of a message.\n\n Like the Generator base class, except that non-text parts are substituted\n with a format string representing the part.\n \"\"\"\n def __init__(self, outfp,\n mangle_from_=True, maxheaderlen=78, fmt=None, linesep=None):\n \"\"\"Like Generator.__init__() except that an additional optional\n argument is allowed.\n\n Walks through all subparts of a message. If the subpart is of main\n type `text', then it prints the decoded payload of the subpart.\n\n Otherwise, fmt is a format string that is used instead of the message\n payload. fmt is expanded with the following keywords (in\n %(keyword)s format):\n\n type : Full MIME type of the non-text part\n maintype : Main MIME type of the non-text part\n subtype : Sub-MIME type of the non-text part\n filename : Filename of the non-text part\n description: Description associated with the non-text part\n encoding : Content transfer encoding of the non-text part\n\n The default value for fmt is None, meaning\n\n [Non-text (%(type)s) part of message omitted, filename %(filename)s]\n \"\"\"\n Generator.__init__(self, outfp, mangle_from_, maxheaderlen, linesep)\n if fmt is None:\n self._fmt = _FMT\n else:\n self._fmt = fmt\n\n def _dispatch(self, msg):\n for part in msg.walk():\n maintype = part.get_content_maintype()\n if maintype == 'text':\n print(part.get_payload(decode=True) + self._NL, end='', file=self)\n elif maintype == 'multipart':\n # Just skip this\n pass\n else:\n print(self._fmt % {\n 'type': part.get_content_type(),\n 'maintype': part.get_content_maintype(),\n 'subtype': part.get_content_subtype(),\n 'filename': part.get_filename('[no filename]'),\n 'description': part.get('Content-Description',\n '[no description]'),\n 'encoding': part.get('Content-Transfer-Encoding',\n '[no encoding]'),\n } + self._NL, end='', file=self)\n\n\n# Helper\n_width = len(repr(sys.maxsize-1))\n_fmt = '%%0%dd' % _width\n\n\ndef _make_boundary(text=None):\n # Craft a random boundary. If text is given, ensure that the chosen\n # boundary doesn't appear in the text.\n token = random.randrange(sys.maxsize)\n boundary = ('=' * 15) + (_fmt % token) + '=='\n if text is None:\n return boundary\n b = boundary\n counter = 0\n while True:\n cre = re.compile('^--' + re.escape(b) + '(--)?$', re.MULTILINE)\n if not cre.search(text):\n break\n b = boundary + '.' + str(counter)\n counter += 1\n return b\n" }, { "path": "mailpile/mailutils/header.py", "content": "# vim: set fileencoding=utf-8 :\n\"\"\" Backport of Python > 3.3 email.header.decode_header()\n\nIt includes fixes that have not been ported to py2\nhttps://bugs.python.org/issue1079\n\n\"\"\"\nfrom __future__ import print_function\nimport binascii\nimport email.quoprimime\nimport email.base64mime\nimport re\nfrom email.errors import HeaderParseError\n\n\n# Match encoded-word strings in the form =?charset?q?Hello_World?=\necre = re.compile(r'''\n =\\? # literal =?\n (?P[^?]*?) # non-greedy up to the next ? is the charset\n \\? # literal ?\n (?P[qb]) # either a \"q\" or a \"b\", case insensitive\n \\? # literal ?\n (?P.*?) # non-greedy up to the next ?= is the encoded string\n \\?= # literal ?=\n ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)\n\n\ndef decode_header(header):\n \"\"\" Decode a message header value without converting charset.\n Returns a list of (string, charset) pairs containing each of the decoded\n parts of the header. Charset is None for non-encoded parts of the header,\n otherwise a lower-case string containing the name of the character set\n specified in the encoded string.\n header may be a string that may or may not contain RFC2047 encoded words,\n or it may be a Header object.\n An email.errors.HeaderParseError may be raised when certain decoding error\n occurs (e.g. a base64 decoding exception).\n \"\"\"\n # If it is a Header object, we can just return the encoded chunks.\n if hasattr(header, '_chunks'):\n return [(_charset._encode(string, str(charset)), str(charset))\n for string, charset in header._chunks]\n # If no encoding, just return the header with no charset.\n if not ecre.search(header):\n return [(header, None)]\n # First step is to parse all the encoded parts into triplets of the form\n # (encoded_string, encoding, charset). For unencoded strings, the last\n # two parts will be None.\n words = []\n for line in header.splitlines():\n parts = ecre.split(line)\n first = True\n while parts:\n unencoded = parts.pop(0)\n if first:\n unencoded = unencoded.lstrip()\n first = False\n if unencoded:\n words.append((unencoded, None, None))\n if parts:\n charset = parts.pop(0).lower()\n encoding = parts.pop(0).lower()\n encoded = parts.pop(0)\n words.append((encoded, encoding, charset))\n # Now loop over words and remove words that consist of whitespace\n # between two encoded strings.\n droplist = []\n for n, w in enumerate(words):\n if n > 1 and w[1] and words[n-2][1] and words[n-1][0].isspace():\n droplist.append(n-1)\n for d in reversed(droplist):\n del words[d]\n\n # The next step is to decode each encoded word by applying the reverse\n # base64 or quopri transformation. decoded_words is now a list of the\n # form (decoded_word, charset).\n decoded_words = []\n for encoded_string, encoding, charset in words:\n if encoding is None:\n # This is an unencoded word.\n decoded_words.append((encoded_string, charset))\n elif encoding == 'q':\n word = email.quoprimime.header_decode(encoded_string)\n decoded_words.append((word, charset))\n elif encoding == 'b':\n # Postel's law: add missing padding\n paderr = len(encoded_string) % 4\n if paderr:\n encoded_string += '==='[:4 - paderr]\n try:\n word = email.base64mime.decode(encoded_string)\n except binascii.Error:\n raise HeaderParseError('Base64 decoding error')\n else:\n decoded_words.append((word, charset))\n else:\n raise AssertionError('Unexpected encoding: ' + encoding)\n # Now convert all words to bytes and collapse consecutive runs of\n # similarly encoded words.\n collapsed = []\n last_word = last_charset = None\n for word, charset in decoded_words:\n if isinstance(word, unicode):\n word = bytes(word, 'raw-unicode-escape')\n if last_word is None:\n last_word = word\n last_charset = charset\n elif charset != last_charset:\n collapsed.append((last_word, last_charset))\n last_word = word\n last_charset = charset\n elif last_charset is None:\n last_word += b' ' + word\n else:\n last_word += word\n collapsed.append((last_word, last_charset))\n return collapsed\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/headerprint.py", "content": "# vim: set fileencoding=utf-8 :\n#\nfrom __future__ import print_function\nimport re\nfrom mailpile.util import md5_hex\n\n\nMUA_ML_HEADERS = (# Mailing lists are sending MUAs in their own right\n 'list-id', 'list-subscribe', 'list-unsubscribe')\n\nMUA_HP_HEADERS = ('date', 'from', 'to', 'reply-to',\n # We omit the Subject, because for some reason it seems\n # to jump around a lot. Same for CC.\n 'message-id', 'return-path', 'precedence', 'organization',\n 'mime-version', 'content-type',\n 'user-agent', 'x-mailer',\n 'x-mimeole', 'x-msmail-priority', 'x-priority',\n 'x-originating-ip', 'x-message-info',\n 'openpgp', 'x-openpgp',\n # Common services\n 'x-github-recipient', 'feedback-id', 'x-facebook')\n\nMUA_ID_HEADERS = ('x-mailer', 'user-agent', 'x-mimeole')\n\nHP_MUA_ID_SPACE = re.compile(r'(\\s+)')\nHP_MUA_ID_IGNORE = re.compile(r'(\\[[a-fA-F0-9%:]+\\]|<\\S+@\\S+>'\n '|(mail|in)-[^\\.]+|\\d+)')\nHP_MUA_ID_SPLIT = re.compile(r'[\\s,/;=()]+')\nHP_RECVD_PARSE = re.compile(r'(by\\s+)'\n '[a-z0-9_\\.-]*?([a-z0-9_-]*?\\.?[a-z0-9_-]+\\s+.*'\n 'with\\s+.*)\\s+id\\s+.*$',\n flags=(re.MULTILINE + re.DOTALL))\n\n\ndef HeaderPrintMTADetails(message):\n \"\"\"Extract details about the sender's outgoing SMTP server.\"\"\"\n details = []\n # We want the first \"non-local\" received line. This can of course be\n # trivially spoofed, but looking at this will still protect against\n # all but the most targeted of spear phishing attacks.\n for rcvd in reversed(message.get_all('received') or []):\n if ('local' not in rcvd\n and ' mapi id ' not in rcvd\n and '127.0.0' not in rcvd\n and '[::1]' not in rcvd):\n parsed = HP_RECVD_PARSE.search(rcvd)\n if parsed:\n by = parsed.group(1) + parsed.group(2)\n by = HP_MUA_ID_SPACE.sub(' ', HP_MUA_ID_IGNORE.sub('x', by))\n details = ['Received ' + by]\n break\n for h in ('DKIM-Signature', 'X-Google-DKIM-Signature'):\n for dkim in (message.get_all(h) or []):\n attrs = [HP_MUA_ID_SPACE.sub('', a)\n for a in dkim.split(';') if a.strip()[:1] in 'vacd']\n details.extend([h, '; '.join(sorted(attrs))])\n return details\n\n\ndef HeaderPrintMUADetails(message, mta=None):\n \"\"\"Summarize what the message tells us directly about the MUA.\"\"\"\n details = []\n for header in MUA_ID_HEADERS:\n value = message.get(header)\n if value:\n # We want some details about the MUA, but also some stability.\n # Thus the HP_MUA_ID_IGNORE regexp...\n value = ' '.join([v for v in HP_MUA_ID_SPLIT.split(value.strip())\n if not HP_MUA_ID_IGNORE.search(v)])\n details.extend([header, value.strip()])\n\n if not details:\n # FIXME: We could definitely make more educated guesses!\n if mta and mta[0].startswith('Received by google.com'):\n details.extend(['Guessed', 'GMail'])\n elif ('x-ms-tnef-correlator' in message or\n 'x-ms-has-attach' in message):\n details.extend(['Guessed', 'Exchange'])\n elif '@mailpile' in message.get('message-id', ''):\n details.extend(['Guessed', 'Mailpile'])\n\n return details\n\n\ndef HeaderPrintGenericDetails(message, which=MUA_HP_HEADERS):\n \"\"\"Extract message details which may help identify the MUA.\"\"\"\n return [k for k, v in message.items() if k.lower() in which]\n\n\ndef HeaderPrints(message):\n \"\"\"Generate fingerprints from message headers which identifies the MUA.\"\"\"\n m = HeaderPrintMTADetails(message)\n u = HeaderPrintMUADetails(message, mta=m)[:20]\n g = HeaderPrintGenericDetails(message)[:50]\n mua = (u[1] if u else None)\n if mua and mua.startswith('Mozilla '):\n mua = mua.split()[-1]\n return {\n # The sender-ID headerprints includes MTA info\n 'sender': md5_hex('\\n'.join(m+u+g)),\n # Tool-chain headerprints ignore the MTA details\n 'tools': md5_hex('\\n'.join(u+g)),\n # Our best guess about what the MUA actually is; may be None\n 'mua': mua}\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/html.py", "content": "# vim: set fileencoding=utf-8 :\n#\nfrom __future__ import print_function\nimport lxml.etree\nimport lxml.html\nimport lxml.html.clean\nimport re\n\n\nRE_HTML_BORING = re.compile(\n '(\\s+|(]*>\\s*)+.*?)',\n flags=re.DOTALL|re.IGNORECASE)\n\nRE_EXCESS_WHITESPACE = re.compile(\n '\\n\\s*\\n\\s*',\n flags=re.DOTALL)\n\nRE_HTML_NEWLINES = re.compile(\n '(]*href=[\\'\"]?([^\\'\">]+)[^>]*>([^<]*)',\n flags=re.DOTALL|re.IGNORECASE)\n\nRE_HTML_IMGS = re.compile(\n ']*src=[\\'\"]?([^\\'\">]+)[^>]*>',\n flags=re.DOTALL|re.IGNORECASE)\n\nRE_HTML_IMG_ALT = re.compile(\n ']*alt=[\\'\"]?([^\\'\">]+)[^>]*>',\n flags=re.DOTALL|re.IGNORECASE)\n\nRE_XML_ENCODING = re.compile(\n '(<\\?xml version=[^ ?>]*((?! +encoding=) [^ ?>]*)*)( +encoding=[^ ?>]*)',\n flags=re.DOTALL|re.IGNORECASE)\n\n\n# FIXME: Decide if this is strict enough or too strict...?\nSHARED_HTML_CLEANER = lxml.html.clean.Cleaner(\n page_structure=True,\n meta=True,\n links=True,\n javascript=True,\n scripts=True,\n frames=True,\n embedded=True,\n safe_attrs_only=True)\n\n\ndef clean_html(html):\n # Find and delete possibly conflicting xml encoding\n # declaration to prevent lxml ValueError.\n # e.g. \n html = re.sub(RE_XML_ENCODING, r'\\1', html).strip()\n return (SHARED_HTML_CLEANER.clean_html(html) if html else '')\n\n\ndef extract_text_from_html(html, url_callback=None):\n try:\n # We compensate for some of the limitations of lxml...\n links, imgs = [], []\n def delink(m):\n url, txt = m.group(1), m.group(2).strip()\n if url_callback is not None:\n url_callback(url, txt)\n if txt[:4] in ('http', 'www.'):\n return txt\n elif url.startswith('mailto:'):\n if '@' in txt:\n return txt\n else:\n return '%s (%s)' % (txt, url.split(':', 1)[1])\n else:\n links.append(' [%d] %s%s' % (len(links) + 1,\n txt and (txt + ': ') or '',\n url))\n return '%s[%d]' % (txt, len(links))\n\n def deimg(m):\n tag, url = m.group(0), m.group(1)\n if ' alt=' in tag:\n return re.sub(RE_HTML_IMG_ALT, '\\1', tag).strip()\n else:\n imgs.append(' [%d] %s' % (len(imgs)+1, url))\n return '[Image %d]' % len(imgs)\n\n html = (\n re.sub(RE_XML_ENCODING, r'\\1',\n re.sub(RE_HTML_PARAGRAPHS, '\\n\\n\\\\1',\n re.sub(RE_HTML_NEWLINES, '\\n\\\\1',\n re.sub(RE_HTML_BORING, ' ',\n re.sub(RE_HTML_LINKS, delink,\n re.sub(RE_HTML_IMGS, deimg, html\n ))))))).strip()\n\n if html:\n try:\n html_text = lxml.html.fromstring(html).text_content()\n except lxml.etree.Error:\n html_text = _('(Invalid HTML suppressed)')\n else:\n html_text = ''\n\n text = (html_text +\n (links and '\\n\\nLinks:\\n' or '') + '\\n'.join(links) +\n (imgs and '\\n\\nImages:\\n' or '') + '\\n'.join(imgs))\n\n return re.sub(RE_EXCESS_WHITESPACE, '\\n\\n', text).strip()\n except:\n import traceback\n traceback.print_exc()\n return html\n\n\nif __name__ == \"__main__\":\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/safe.py", "content": "from __future__ import print_function\nimport email\nimport email.errors\nimport email.message\nimport random\nimport re\nimport rfc822\nimport time\nfrom urllib import quote, unquote\n\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailutils.header import decode_header\nfrom mailpile.util import *\n\n\ndef safe_decode_hdr(msg=None, name=None, hdr=None, charset=None):\n \"\"\"\n This method stubbornly tries to decode header data and convert\n to Pythonic unicode strings. The strings are guaranteed not to\n contain tab, newline or carriage return characters.\n\n If used with a message object, the header and the MIME charset\n will be inferred from the message headers.\n >>> msg = email.message.Message()\n >>> msg['content-type'] = 'text/plain; charset=utf-8'\n >>> msg['from'] = 'G\\\\xc3\\\\xadsli R \\\\xc3\\\\x93la '\n >>> safe_decode_hdr(msg, 'from')\n u'G\\\\xedsli R \\\\xd3la '\n\n The =?...?= MIME header encoding is also recognized and processed.\n\n >>> safe_decode_hdr(hdr='=?iso-8859-1?Q?G=EDsli_R_=D3la?=\\\\r\\\\n')\n u'G\\\\xedsli R \\\\xd3la '\n\n >>> safe_decode_hdr(hdr='\"=?utf-8?Q?G=EDsli_R?= =?iso-8859-1?Q?=D3la?=\"')\n u'G\\\\xedsli R \\\\xd3la'\n\n And finally, guesses are made with raw binary data. This process\n could be improved, it currently only attempts utf-8 and iso-8859-1.\n\n >>> safe_decode_hdr(hdr='\"G\\\\xedsli R \\\\xd3la\"\\\\r\\\\t')\n u'\"G\\\\xedsli R \\\\xd3la\" '\n\n >>> safe_decode_hdr(hdr='\"G\\\\xc3\\\\xadsli R \\\\xc3\\\\x93la\"\\\\n ')\n u'\"G\\\\xedsli R \\\\xd3la\" '\n\n # See https://bugs.python.org/issue1079\n\n # encoded word enclosed in parenthesis (comment syntax)\n >>> safe_decode_hdr(hdr='rene@example.com (=?utf-8?Q?Ren=C3=A9?=)')\n u'rene@example.com ( Ren\\\\xe9 )'\n\n # no space after encoded word\n >>> safe_decode_hdr(hdr='=?UTF-8?Q?Direction?=')\n u'Direction '\n \"\"\"\n if hdr is None:\n value = msg and msg[name] or ''\n charset = charset or msg.get_content_charset() or 'utf-8'\n else:\n value = hdr\n charset = charset or 'utf-8'\n\n if not isinstance(value, unicode):\n # Already a str! Oh shit, might be nasty binary data.\n value = try_decode(value, charset, replace='?')\n\n # At this point we know we have a unicode string. Next we try\n # to very stubbornly decode and discover character sets.\n if '=?' in value and '?=' in value:\n try:\n # decode_header wants an unquoted str (not unicode)\n value = value.encode('utf-8').replace('\"', '')\n # Decode!\n pairs = decode_header(value)\n value = ' '.join([try_decode(t, cs or charset)\n for t, cs in pairs])\n except email.errors.HeaderParseError:\n pass\n\n # Finally, return the unicode data, with white-space normalized\n return value.replace('\\r', ' ').replace('\\t', ' ').replace('\\n', ' ')\n\ndef safe_parse_date(date_hdr):\n \"\"\"Parse a Date: or Received: header into a unix timestamp.\"\"\"\n try:\n if ';' in date_hdr:\n date_hdr = date_hdr.split(';')[-1].strip()\n msg_ts = long(rfc822.mktime_tz(rfc822.parsedate_tz(date_hdr)))\n if (msg_ts > (time.time() + 24 * 3600)) or (msg_ts < 1):\n return None\n else:\n return msg_ts\n except (ValueError, TypeError, OverflowError):\n return None\n\ndef safe_message_ts(msg, default=None, msg_mid=None, msg_id=None, session=None):\n \"\"\"Extract a date, sanity checking against the Received: headers.\"\"\"\n hdrs = [safe_decode_hdr(msg, 'date')] + (msg.get_all('received') or [])\n dates = [safe_parse_date(date_hdr) for date_hdr in hdrs]\n msg_ts = dates[0]\n nz_dates = sorted([d for d in dates if d])\n\n if nz_dates:\n a_week = 7 * 24 * 3600\n\n # Ideally, we compare with the date on the 2nd SMTP relay, as\n # the first will often be the same host as composed the mail\n # itself. If we don't have enough hops, just use the last one.\n #\n # We don't want to use a median or average, because if the\n # message bounces around lots of relays or gets resent, we\n # want to ignore the latter additions.\n #\n rcv_ts = nz_dates[min(len(nz_dates)-1, 2)]\n\n # Now, if everything is normal, the msg_ts will be at nz_dates[0]\n # and it won't be too far away from our reference date.\n if (msg_ts == nz_dates[0]) and (abs(msg_ts - rcv_ts) < a_week):\n # Note: Trivially true for len(nz_dates) in (1, 2)\n return msg_ts\n\n # Damn, dates are screwy!\n #\n # Maybe one of the SMTP servers has a wrong clock? If the Date:\n # header falls within the range of all detected dates (plus a\n # week towards the past), still trust it.\n elif ((msg_ts >= (nz_dates[0]-a_week))\n and (msg_ts <= nz_dates[-1])):\n return msg_ts\n\n # OK, Date: is insane, use one of the early Received: lines\n # instead. We picked the 2nd one above, that should do.\n else:\n if session and msg_mid and msg_id:\n session.ui.warning(_('=%s/%s using Received: instead of Date:'\n ) % (msg_mid, msg_id))\n return rcv_ts\n else:\n # If the above fails, we assume the messages in the mailbox are in\n # chronological order and just add 1 second to the date of the last\n # message if date parsing fails for some reason.\n if session and msg_mid and msg_id:\n session.ui.warning(_('=%s/%s has a bogus date'\n ) % (msg_mid, msg_id))\n return default\n\ndef safe_get_msg_id(msg):\n raw_msg_id = safe_decode_hdr(msg, 'message-id')\n if not raw_msg_id:\n # Create a very long pseudo-msgid for messages without a\n # Message-ID. This was a very badly behaved mailer, so if\n # we create duplicates this way, we are probably only\n # losing spam. Even then the Received line should save us.\n raw_msg_id = ('\\t'.join([safe_decode_hdr(msg, 'date'),\n safe_decode_hdr(msg, 'subject'),\n safe_decode_hdr(msg, 'received'),\n safe_decode_hdr(msg, 'from'),\n safe_decode_hdr(msg, 'to')])\n # This is to avoid truncation in encode_msg_id:\n ).replace('<', '').strip()\n return raw_msg_id\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/mailutils/vcal.py", "content": "from __future__ import print_function\nimport time\nimport icalendar\nfrom datetime import datetime\n\ndef calendar_parse(payload):\n c = icalendar.parser.Contentlines()\n lines = c.from_ical(payload)\n\n root = None\n obj = None\n\n for line in lines:\n if line == \"\":\n break\n parts = line.parts()\n if parts[0] == \"BEGIN\":\n t = vmap[parts[2]]()\n if not root: root = t\n if obj:\n obj.children.append(t)\n t.parent = obj\n obj = t\n\n root.stack.append(parts[2])\n continue\n if parts[0] == \"END\":\n obj = obj.parent\n root.stack.pop()\n continue\n\n obj.add_part(*parts)\n\n return root.to_json()\n\nclass VObject:\n def __init__(self):\n self.children = []\n self.parent = None\n self.stack = []\n self.parts = []\n\n def add_part(self, key, params, value):\n self.parts.append([key, params, value])\n\n def find_parts(self, key):\n res = []\n for p in self.parts:\n if p[0] == key:\n res.append(p)\n return res\n\n def find_one_part(self, key):\n res = self.find_parts(key)\n if len(res) == 0: return None\n r = {\"value\": res[0][2], \"params\": res[0][1] }\n return r\n\n def find_one_part_value(self, key, value=None):\n res = self.find_parts(key)\n if len(res) == 0: return value\n return res[0][2]\n\n def get_datetime(self, key):\n val = self.find_one_part_value(key)\n try:\n return datetime(*time.strptime(val, \"%Y%m%dT%H%M%SZ\")[:6])\n except:\n return datetime(*time.strptime(val, \"%Y%m%dT%H%M%S\")[:6])\n\n def to_raw_json(self):\n parts = {}\n for p in self.parts:\n if p[0] not in parts:\n parts[p[0]] = []\n parts[p[0]].append({\"value\": p[2], \"parameters\": p[1]})\n\n children = [x.to_raw_json() for x in self.children]\n return {\n \"type\": self.__class__.__name__,\n \"children\": children,\n \"parts\": parts,\n }\n\n def to_json(self):\n return to_raw_json()\n\nclass VTimeZone(VObject):\n pass\n\nclass VTZStandard(VObject):\n pass\n\nclass VTZDaylight(VObject):\n pass\n\nclass VAlarm(VObject):\n pass\n\nclass VEvent(VObject):\n def __init__(self):\n VObject.__init__(self)\n\n def to_json(self):\n summary = self.find_one_part_value(\"SUMMARY\", \"\")\n description = self.find_one_part_value(\"DESCRIPTION\", \"\").replace(\"\\\\n\", \"\\n\").replace(\"\\n\\n\", \"\\n\")\n dtstart = self.get_datetime(\"DTSTART\")\n dtend = self.get_datetime(\"DTEND\")\n location = self.find_one_part_value(\"LOCATION\", \"\")\n attendees = [{\"cn\": x[1][\"cn\"], \"email\": x[2].split(\":\")[1]}\n for x in self.find_parts(\"ATTENDEE\")]\n o = self.find_one_part(\"ORGANIZER\")\n organizer = { \"cn\": o[\"params\"][\"CN\"], \"email\": o[\"value\"].split(\":\")[1]}\n tzinfo = None\n\n return {\n \"summary\": summary,\n \"description\": description,\n \"dtstart\": dtstart,\n \"dtend\": dtend,\n \"location\": location,\n \"timezone\": tzinfo,\n \"organizer\": organizer,\n \"attendees\": attendees,\n \"alarms\": [],\n }\n\nclass VCalendar(VObject):\n def __init__(self):\n VObject.__init__(self)\n\n def print_events(self):\n for e in self.children:\n if isinstance(e, VEvent):\n print(\"%s invited you to %s\" % (e.find_parts(\"ORGANIZER\")[0][1]['CN'],\n e.find_parts(\"SUMMARY\")[0][2]))\n print(\"%s\" % e.find_parts(\"DTSTART\")[0][2])\n print(\"%s\" % e.find_parts(\"LOCATION\")[0][2])\n\n def to_json(self):\n events = []\n for e in self.children:\n # We are assuming VEvents will only occur immediately under the\n # VCalendar level. Haven't seen anything else in the wild.\n if isinstance(e, VEvent):\n events.append(e.to_json())\n\n return events\n\n\nvmap = {\n \"VALARM\": VAlarm,\n \"VTIMEZONE\": VTimeZone,\n \"VEVENT\": VEvent,\n \"VCALENDAR\": VCalendar,\n \"STANDARD\": VTZStandard,\n \"DAYLIGHT\": VTZDaylight,\n}\n\nif __name__ == \"__main__\":\n cal = calendar_parse(open(\"calitem.cal\").read())\n # cal.print_tree()\n print(\"------------------------------\")\n cal.print_events()\n print(\"------------------------------\")\n" }, { "path": "mailpile/packing.py", "content": "from __future__ import print_function\nimport struct\nimport time\nimport zlib\n\nfrom mailpile.util import *\n\n\ndef PackIntSet(ints):\n \"\"\"\n Pack a set of ints to a compact string, unpackable by UnpackIntSet.\n\n Short lists are binary packed directly, but long lists are converted\n to a bitmask and then compressed using zlib.\n\n >>> intset = set([1, 5, 9, 10000])\n >>> intsetstr = PackIntSet(intset)\n >>> type(intsetstr), len(intsetstr)\n (, 16)\n\n >>> UnpackIntSet(intsetstr) == intset\n True\n\n >>> intset = set(list(range(1000, 50000) + [1, 2, 3]))\n >>> intsetstr = PackIntSet(intset)\n >>> intsetstr.startswith('\\xff\\xff\\xff\\xff')\n True\n\n >>> len(intsetstr)\n 37\n\n >>> UnpackIntSet(intsetstr) == intset\n True\n \"\"\"\n if len(ints) > 15:\n return '\\xff\\xff\\xff\\xff' + zlib.compress(intlist_to_bitmask(ints))\n else:\n return struct.pack('<' + 'I' * len(ints), *ints)\n\n\ndef UnpackIntSet(data):\n \"\"\"\n Unpack a set of ints previously packed using PackIntSet.\n \"\"\"\n if len(data) > 13 and data[:4] == '\\xff\\xff\\xff\\xff':\n return set(bitmask_to_intlist(zlib.decompress(data[4:])))\n else:\n return set(struct.unpack('<' + 'I' * (len(data)//4), data))\n\n\ndef PackLongList(longs):\n \"\"\"\n Pack a list of longs to a compact string, unpackable by UnpackLongList.\n\n Short lists are binary packed directly:\n >>> ll = [1, 5, 100000000000L]\n >>> llstr = PackLongList(ll)\n\n >>> UnpackLongList(llstr) == ll\n True\n\n >>> type(llstr), len(llstr)\n (, 24)\n\n Longer lists are zlib compressed, which can result in significant space\n savings for many types of data.\n >>> ll += list(range(100, 1000))\n >>> llstr = PackLongList(ll)\n >>> llstr.startswith('\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff')\n True\n\n >>> UnpackLongList(llstr) == ll\n True\n\n >>> len(llstr)\n 1416\n \"\"\"\n packed = struct.pack('<' + 'q' * len(longs), *longs)\n if (len(packed) > 8 * 15) or (longs[0] == 0xffffffffffffffff):\n return ('\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff' + zlib.compress(packed))\n else:\n return packed\n\n\ndef UnpackLongList(data):\n \"\"\"\n Unpack a list of longs previously packed using PackLongList.\n \"\"\"\n if len(data) > 17 and data[:8] == '\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff':\n data = zlib.decompress(data[8:])\n return list(struct.unpack('<' + 'q' * (len(data)//8), data))\n\n\nclass StorageBackedData(object):\n \"\"\"\n This lovely hack exposes the full API of a Python set or list, but any\n writes get flushed to a storage backend and the initial state is loaded\n from the same.\n\n It is NOT SAFE to ever have more than one of these for a given backend\n as they will not stay in sync. Since most methods are proxies, using\n set method on a backed list will fail and vice-versa.\n\n This class must be subclassed and _pack and _unpack implemented.\n \"\"\"\n def __init__(self, storage, skey):\n self._storage = storage\n self._skey = skey\n self.load()\n self.last_save = time.time()\n self.auto_save = True\n self.interval = -1\n self.dirty = False\n\n def _pack(self, data): raise NotImplemented()\n def _unpack(self, data): raise NotImplemented()\n\n def load(self):\n try:\n self._obj = self._unpack(self._storage[self._skey])\n except (KeyError, IndexError):\n self._obj = self._unpack('')\n\n def save(self, maybe=False):\n if not maybe or self.dirty:\n self._storage[self._skey] = self._pack(self._obj)\n self.dirty = False\n\n def _dirty_maybe_save(self):\n self.dirty = True\n if self.auto_save:\n if (self.interval < 1 or\n self.last_save < time.time() - self.interval):\n self.save()\n\n def _r(self, method, *args, **kwargs):\n return getattr(self._obj, method)(*args, **kwargs)\n\n def _w(self, method, *args, **kwargs):\n rv = getattr(self._obj, method)(*args, **kwargs)\n self._dirty_maybe_save()\n return rv\n\n def _iw(self, method, *args, **kwargs):\n self._obj = getattr(self._obj, method)(*args, **kwargs)\n self._dirty_maybe_save()\n return self\n \n def __and__(s, *a, **kw): return s._r('__and__', *a, **kw)\n def __cmp__(s, *a, **kw): return s._r('__cmp__', *a, **kw)\n def __contains__(s, *a, **kw): return s._r('__contains__', *a, **kw)\n def __eq__(s, *a, **kw): return s._r('__eq__', *a, **kw)\n def __ge__(s, *a, **kw): return s._r('__ge__', *a, **kw)\n def __getitem__(s, *a, **kw): return s._r('__getitem__', *a, **kw)\n def __getslice__(s, *a, **kw): return s._r('__getslice__', *a, **kw)\n def __gt__(s, *a, **kw): return s._r('__gt__', *a, **kw)\n def __iter__(s, *a, **kw): return s._r('__iter__', *a, **kw)\n def __le__(s, *a, **kw): return s._r('__le__', *a, **kw)\n def __len__(s, *a, **kw): return s._r('__len__', *a, **kw)\n def __lt__(s, *a, **kw): return s._r('__lt__', *a, **kw)\n def __mul__(s, *a, **kw): return s._r('__mul__', *a, **kw)\n def __ne__(s, *a, **kw): return s._r('__ne__', *a, **kw)\n def __or__(s, *a, **kw): return s._r('__or__', *a, **kw)\n def __rand__(s, *a, **kw): return s._r('__rand__', *a, **kw)\n def __reduce__(s, *a, **kw): return s._r('__reduce__', *a, **kw)\n def __repr__(s, *a, **kw): return s._r('__repr__', *a, **kw)\n def __reversed__(s, *a, **kw): return s._r('__reversed__', *a, **kw)\n def __rmul__(s, *a, **kw): return s._r('__rmul__', *a, **kw)\n def __rsub__(s, *a, **kw): return s._r('__rsub__', *a, **kw)\n def __rxor__(s, *a, **kw): return s._r('__rxor__', *a, **kw)\n def __sizeof__(s, *a, **kw): return s._r('__sizeof__', *a, **kw)\n def __sub__(s, *a, **kw): return s._r('__sub__', *a, **kw)\n def __xor__(s, *a, **kw): return s._r('__xor__', *a, **kw)\n def copy(s, *a, **kw): return s._r('copy', *a, **kw)\n def count(s, *a, **kw): return s._r('count', *a, **kw)\n def difference(s, *a, **kw): return s._r('difference', *a, **kw)\n def index(s, *a, **kw): return s._r('index', *a, **kw)\n def intersection(s, *a, **kw): return s._r('intersection', *a, **kw)\n def isdisjoint(s, *a, **kw): return s._r('isdisjoint', *a, **kw)\n def issubset(s, *a, **kw): return s._r('issubset', *a, **kw)\n def issuperset(s, *a, **kw): return s._r('issuperset', *a, **kw)\n def union(s, *a, **kw): return s._r('union', *a, **kw)\n\n def symmetric_difference(s, *a, **kw):\n return s._r('symmetric_difference', *a, **kw)\n\n def __iadd__(s, *a, **kw): return s._iw('__iadd__', *a, **kw)\n def __iand__(s, *a, **kw): return s._iw('__iand__', *a, **kw)\n def __imul__(s, *a, **kw): return s._iw('__imul__', *a, **kw)\n def __ior__(s, *a, **kw): return s._iw('__ior__', *a, **kw)\n def __isub__(s, *a, **kw): return s._iw('__isub__', *a, **kw)\n def __ixor__(s, *a, **kw): return s._iw('__ixor__', *a, **kw)\n\n def __delitem__(s, *a, **kw): return s._w('__delitem__', *a, **kw)\n def __delslice__(s, *a, **kw): return s._w('__delslice__', *a, **kw)\n def __setitem__(s, *a, **kw): return s._w('__setitem__', *a, **kw)\n def __setslice__(s, *a, **kw): return s._w('__setslice__', *a, **kw)\n def add(s, *a, **kw): return s._w('add', *a, **kw)\n def append(s, *a, **kw): return s._w('append', *a, **kw)\n def clear(s, *a, **kw): return s._w('clear', *a, **kw)\n def discard(s, *a, **kw): return s._w('discard', *a, **kw)\n def extend(s, *a, **kw): return s._w('extend', *a, **kw)\n def insert(s, *a, **kw): return s._w('insert', *a, **kw)\n def pop(s, *a, **kw): return s._w('pop', *a, **kw)\n def remove(s, *a, **kw): return s._w('remove', *a, **kw)\n def reverse(s, *a, **kw): return s._w('reverse', *a, **kw)\n def sort(s, *a, **kw): return s._w('sort', *a, **kw)\n def update(s, *a, **kw): return s._w('update', *a, **kw)\n\n def difference_update(s, *a, **kw):\n return s._w('difference_update', *a, **kw)\n def intersection_update(s, *a, **kw):\n return s._w('intersection_update', *a, **kw)\n def symmetric_difference_update(s, *a, **kw):\n return s._w('symmetric_difference_update', *a, **kw)\n\n\nclass StorageBackedSet(StorageBackedData):\n \"\"\"\n This combines StorageBackedData with Pack/UnpackIntSet to pack\n and save sets of ints.\n\n >>> storage = {'sbs': '\\\\x01\\\\x00\\\\x00\\\\x00'}\n >>> sbs = StorageBackedSet(storage, 'sbs')\n >>> 1 in sbs\n True\n\n >>> sbs.add(2)\n >>> sbs.save()\n >>> UnpackIntSet(storage['sbs']) == set([1, 2])\n True\n \"\"\"\n def _pack(self, data): return PackIntSet(data)\n def _unpack(self, data): return UnpackIntSet(data)\n\n\nclass StorageBackedLongs(StorageBackedData):\n \"\"\"\n This combines StorageBackedData with Pack/UnpackLongList to pack\n and save sets of ints.\n\n >>> storage = {'sbl': '\\\\x01\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00'}\n >>> sbl = StorageBackedLongs(storage, 'sbl')\n >>> 1 in sbl\n True\n\n >>> sbl.append(2)\n >>> sbl.save()\n >>> UnpackLongList(storage['sbl']) == [1, 2]\n True\n \"\"\"\n def _pack(self, data): return PackLongList(data)\n def _unpack(self, data): return UnpackLongList(data)\n\n\nif __name__ == '__main__':\n import doctest\n import sys\n results = doctest.testmod(optionflags=doctest.ELLIPSIS,\n extraglobs={})\n print('%s' % (results, ))\n if results.failed:\n sys.exit(1)\n" }, { "path": "mailpile/platforms.py", "content": "\"\"\"\nThis module tries to centralize most of the platform-specific code in use by\nMailpile. If you find yourself checking which platform the app runs on, adding\na function here instead is probably The Right Thing.\n\"\"\"\nimport copy\nimport os\nimport subprocess\nimport sys\n\n\n# This is a cache of discovered binaries and their paths.\nBINARIES = {}\n\n\n# These are the binaries we want, and the test we use to detect whether\n# they are available/working.\n#\nBINARIES_WANTED = {# Test command Required?\n 'GnuPG': (['gpg', '--version'], True),\n 'GnuPG_dirmngr': (['dirmngr', '--version'], True),\n 'GnuPG_agent': (['gpg-agent', '--version'], True),\n 'OpenSSL': (['openssl', 'version'], True),\n 'Tor': (['tor', '--version'], False)}\n\n\ndef _assert_file_exists(path):\n if not os.path.exists(path):\n raise OSError('Not found: %s' % path)\n return path\n\n\ndef DetectBinaries(\n which=None, use_cache=True, preferred={}, skip=None, _raise=None):\n import mailpile.util\n import mailpile.safe_popen\n import traceback\n\n global BINARIES\n if which and use_cache:\n if which in BINARIES:\n return BINARIES[which]\n env_bin = os.getenv('MAILPILE_%s' % which.upper(), '')\n if env_bin:\n BINARIES[which] = env_bin\n return env_bin\n\n if skip is None:\n skip = (os.getenv('MAILPILE_IGNORE_BINARIES', '')\n .replace('/ga', '_agent') # Backwards compatibility\n .replace('/dm', '_dirmngr') # Backwards compatibility\n .split())\n\n def _run_bintest(bt):\n p = mailpile.safe_popen.Popen(bt,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n return p.communicate()\n\n for binary, (bin_test, reqd) in BINARIES_WANTED.iteritems():\n if binary in skip:\n continue\n if (which is None) or (binary == which):\n if preferred.get(binary):\n bin_test = copy.copy(bin_test)\n bin_test[0] = preferred[binary]\n else:\n env_bin = os.getenv('MAILPILE_%s' % binary.upper(), '')\n if env_bin:\n BINARIES[binary] = env_bin\n continue\n try:\n mailpile.util.RunTimed(5.0, _run_bintest, bin_test)\n BINARIES[binary] = bin_test[0]\n if (not os.path.dirname(BINARIES[binary])\n and not sys.platform.startswith('win')):\n try:\n path = subprocess.check_output(['which',\n BINARIES[binary]])\n if path:\n BINARIES[binary] = path.strip()\n except (OSError, subprocess.CalledProcessError):\n pass\n except (OSError, subprocess.CalledProcessError, mailpile.util.TimedOut):\n if binary in BINARIES:\n del BINARIES[binary]\n\n if which:\n if _raise not in (None, False):\n if not BINARIES.get(which):\n raise _raise('%s not found' % which)\n return BINARIES.get(which)\n\n elif _raise not in (None, False):\n for binary, (bin_test, reqd) in BINARIES_WANTED.iteritems():\n if binary in skip or not reqd:\n continue\n if not BINARIES.get(binary):\n raise _raise('%s not found' % binary)\n\n return BINARIES\n\n\ndef GetDefaultGnuPGCommand(_raise=OSError):\n return DetectBinaries(which='GnuPG', _raise=_raise)\n\n\ndef GetDefaultOpenSSLCommand(_raise=OSError):\n return DetectBinaries(which='OpenSSL', _raise=_raise)\n\n\ndef GetDefaultTorPath(_raise=OSError):\n return DetectBinaries(which='Tor', _raise=_raise)\n\n\ndef InDesktopEnvironment():\n \"\"\"\n Returns True if we're running in a desktop environment of some sort.\n \"\"\"\n # FIXME: Detect if we are somehow in the background on Windows or OS X.\n return (sys.platform[:3] in ('dar', 'win') or os.getenv('DISPLAY'))\n\n\ndef RenameCannotOverwrite():\n \"\"\"\n The os.rename() function will not overwrite existing files on Windows.\n \"\"\"\n return sys.platform.startswith('win')\n\n\ndef NeedExplicitPortCheck():\n \"\"\"\n Our HTTP worker doesn't detect port reuse on Windows, need explicit checks.\n \"\"\"\n return sys.platform.startswith('win')\n\n\ndef TerminalSupportsAnsiColors():\n \"\"\"\n Windows doesn't like ANSI colors. Also, we want a TTY.\n \"\"\"\n return (sys.stdout.isatty() and sys.platform[:3] != \"win\")\n\n\ndef WindowsPopenSemantics():\n \"\"\"\n The safe_popen module implements slightly different semantics on Windows.\n \"\"\"\n return sys.platform.startswith('win')\n\n\ndef GetAppDataDirectory():\n if sys.platform.startswith('win'):\n # Obey Windows conventions (more or less?)\n return os.getenv('APPDATA', os.path.expanduser('~'))\n elif sys.platform.startswith('darwin'):\n # Obey Mac OS X conventions\n return os.path.expanduser('~/Library/Application Support')\n else:\n # Assume other platforms are Unixy\n return os.getenv('XDG_DATA_HOME', os.path.expanduser('~/.local/share'))\n\n\ndef RestrictReadAccess(path):\n \"\"\"\n Restrict access to a file or directory so only the user can read it.\n \"\"\"\n # FIXME: Windows code goes here!\n if os.path.isdir(path):\n os.chmod(path, 0o700)\n else:\n os.chmod(path, 0o600)\n\n\ndef RandomListeningPort(count=1, host='127.0.0.1'):\n socks = []\n ports = []\n try:\n import socket\n for port in range(0, count):\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n sock.bind((host, 0))\n socks.append(sock)\n ports.append(sock.getsockname()[1])\n if count == 1:\n return ports[0]\n else:\n return ports\n finally:\n for sock in socks:\n sock.close()\n" }, { "path": "mailpile/plugins/__init__.py", "content": "from __future__ import print_function\n# Plugins!\nimport imp\nimport inspect\nimport json\nimport os\nimport sys\nimport traceback\n\nimport mailpile.commands\nimport mailpile.config.defaults\nimport mailpile.vcard\nfrom mailpile.i18n import i18n_disabled\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import register as register_mailbox\nfrom mailpile.util import *\n\n\n##[ Plugin discovery ]########################################################\n\n\n# These are the plugins we ship/import by default\n__all__ = [\n 'core',\n 'eventlog', 'search', 'tags', 'contacts', 'compose', 'groups',\n 'dates', 'sizes', 'autotag', 'cryptostate', 'crypto_gnupg', 'gui',\n 'setup_magic', 'oauth', 'exporters', 'plugins', 'motd', 'backups',\n 'vcard_carddav', 'vcard_gnupg', 'vcard_gravatar', 'vcard_libravatar',\n 'vcard_mork', 'html_magic', 'migrate', 'smtp_server', 'crypto_policy',\n 'keylookup', 'webterminal', 'crypto_autocrypt'\n]\nPLUGINS = __all__\n\n\nclass EmailTransform(object):\n \"\"\"Base class for e-mail transforms\"\"\"\n def __init__(self, config):\n self.config = config\n\n def _get_sender_profile(self, sender, kwargs):\n profile = kwargs.get('sender_profile')\n if not profile:\n profile = self.config.get_profile(sender)\n return profile\n\n def _get_first_part(self, msg, mimetype):\n for part in msg.walk():\n if not part.is_multipart():\n mimetype = (part.get_content_type() or 'text/plain').lower()\n if mimetype == 'text/plain':\n return part\n return None\n\n def TransformIncoming(self, *args, **kwargs):\n return list(args[:]) + [False]\n\n def TransformOutgoing(self, *args, **kwargs):\n return list(args[:]) + [False, True]\n\n\nclass PluginError(Exception):\n pass\n\n\nclass PluginManager(object):\n \"\"\"\n Manage importing and loading of plugins. Note that this class is\n effectively a singleton, as it works entirely with globals within\n the mailpile.plugins module.\n \"\"\"\n DEFAULT = __all__\n BUILTIN = (DEFAULT + [\n 'autotag_sb'\n ])\n\n # These are plugins which we consider required\n REQUIRED = [\n 'core',\n 'eventlog', 'search', 'tags', 'contacts', 'compose', 'groups',\n 'dates', 'sizes', 'cryptostate', 'setup_magic', 'oauth', 'html_magic',\n 'plugins', 'keylookup', 'motd', 'backups', 'gui'\n ]\n # Plugins we want, if they are discovered\n WANTED = [\n 'autoajax', 'print', 'hints'\n ]\n # Plugins that have been renamed from past releases\n RENAMED = {\n 'crypto_utils': 'crypto_gnupg'\n }\n DISCOVERED = {}\n LOADED = []\n\n def __init__(self, plugin_name=None, builtin=False, deprecated=False,\n config=None, session=None):\n if builtin and isinstance(builtin, (str, unicode)):\n builtin = os.path.basename(builtin)\n for ignore in ('.py', '.pyo', '.pyc'):\n if builtin.endswith(ignore):\n builtin = builtin[:-len(ignore)]\n if builtin not in self.LOADED:\n self.LOADED.append(builtin)\n\n self.loading_plugin = plugin_name\n self.loading_builtin = plugin_name and builtin\n self.builtin = builtin\n self.deprecated = deprecated\n self.session = session\n self.config = config\n self.manifests = []\n\n def _listdir(self, path):\n try:\n return [d for d in os.listdir(path) if not d.startswith('.')]\n except OSError:\n return []\n\n def _uncomment(self, json_data):\n return '\\n'.join([l for l in json_data.splitlines()\n if not l.strip().startswith('#')])\n\n def discover(self, paths, update=False):\n \"\"\"\n Scan the plugin directories for plugins we could load.\n This updates the global PluginManager state and returns the\n PluginManager itself (for chaining).\n \"\"\"\n plugins = self.BUILTIN[:]\n for pdir in paths:\n for subdir in self._listdir(pdir):\n pname = subdir.lower()\n if pname in self.BUILTIN:\n print('Cannot overwrite built-in plugin: %s' % pname)\n continue\n if pname in self.DISCOVERED and not update:\n # FIXME: this is lame\n # print 'Ignoring duplicate plugin: %s' % pname\n continue\n plug_path = os.path.join(pdir, subdir)\n manifest_filename = os.path.join(plug_path, 'manifest.json')\n try:\n with open(manifest_filename) as mfd:\n manifest = json.loads(self._uncomment(mfd.read()))\n safe_assert(manifest.get('name') == subdir)\n # FIXME: Need more sanity checks\n self.DISCOVERED[pname] = (plug_path, manifest)\n except (ValueError, AssertionError):\n print('Bad manifest: %s' % manifest_filename)\n except (OSError, IOError):\n pass\n\n return self\n\n def available(self):\n return self.BUILTIN[:] + self.DISCOVERED.keys()\n\n def loadable(self):\n return self.BUILTIN[:] + self.RENAMED.keys() + self.DISCOVERED.keys()\n\n def loadable_early(self):\n return [k for k, (n, m) in self.DISCOVERED.iteritems()\n if not m.get('require_login', True)]\n\n def _import(self, full_name, full_path):\n # create parents as necessary\n parents = full_name.split('.')[2:] # skip mailpile.plugins\n module = \"mailpile.plugins\"\n for parent in parents:\n mp = '%s.%s' % (module, parent)\n if mp not in sys.modules:\n sys.modules[mp] = imp.new_module(mp)\n sys.modules[module].__dict__[parent] = sys.modules[mp]\n module = mp\n safe_assert(module == full_name)\n\n # load actual module\n sys.modules[full_name].__file__ = full_path\n with i18n_disabled:\n with open(full_path, 'r') as mfd:\n exec(mfd.read(), sys.modules[full_name].__dict__)\n\n def _load(self, plugin_name, process_manifest=False, config=None):\n full_name = 'mailpile.plugins.%s' % plugin_name\n if full_name in sys.modules:\n return self\n\n self.loading_plugin = full_name\n if plugin_name in self.BUILTIN:\n # The builtins are just normal Python code. If they have a\n # manifest, they'll invoke process_manifest themselves.\n self.loading_builtin = True\n module = __import__(full_name)\n\n elif plugin_name in self.DISCOVERED:\n dirname, manifest = self.DISCOVERED[plugin_name]\n self.loading_builtin = False\n\n # Load the Python requested by the manifest.json\n files = manifest.get('code', {}).get('python', [])\n try:\n for filename in files:\n path = os.path.join(dirname, filename)\n if filename == '.':\n self._import(full_name, dirname)\n continue\n elif filename.endswith('.py'):\n subname = filename[:-3].replace('/', '.')\n # FIXME: Is this a good idea?\n if full_name.endswith('.'+subname):\n self._import(full_name, path)\n continue\n elif os.path.isdir(path):\n subname = filename.replace('/', '.')\n else:\n continue\n self._import('.'.join([full_name, subname]), path)\n except KeyboardInterrupt:\n raise\n except:\n traceback.print_exc(file=sys.stderr)\n print('FIXME: Loading %s failed, tell user!' % full_name)\n if full_name in sys.modules:\n del sys.modules[full_name]\n return None\n\n spec = (full_name, manifest, dirname)\n self.manifests.append(spec)\n if process_manifest:\n self._process_manifest_pass_one(*spec)\n self._process_manifest_pass_two(*spec)\n self._process_startup_hooks(*spec)\n else:\n print('Unrecognized plugin: %s' % plugin_name)\n return self\n\n if plugin_name not in self.LOADED:\n self.LOADED.append(plugin_name)\n return self\n\n def load(self, *args, **kwargs):\n try:\n return self._load(*args, **kwargs)\n finally:\n self.loading_plugin = None\n self.loading_builtin = False\n\n def process_shutdown_hooks(self):\n for plugin_name in self.DISCOVERED.keys():\n try:\n package = 'mailpile.plugins.%s' % plugin_name\n _, manifest = self.DISCOVERED[plugin_name]\n\n if package in sys.modules:\n for method_name in self._mf_path(manifest,\n 'lifecycle', 'shutdown'):\n method = self._get_method(package, method_name)\n method(self.config)\n except:\n # ignore exceptions here as mailpile is going to shut down\n traceback.print_exc(file=sys.stderr)\n\n def process_manifests(self):\n failed = []\n for process in (self._process_manifest_pass_one,\n self._process_manifest_pass_two,\n self._process_startup_hooks):\n for spec in self.manifests:\n try:\n if spec[0] not in failed:\n process(*spec)\n except Exception as e:\n print('Failed to process manifest for %s: %s' % (spec[0], e))\n failed.append(spec[0])\n traceback.print_exc()\n return self\n\n def _mf_path(self, mf, *path):\n for p in path:\n mf = mf.get(p, {})\n return mf\n\n def _mf_iteritems(self, mf, *path):\n return self._mf_path(mf, *path).iteritems()\n\n def _get_method(self, full_name, method):\n full_method_name = '.'.join([full_name, method])\n package, method_name = full_method_name.rsplit('.', 1)\n\n module = sys.modules[package]\n return getattr(module, method_name)\n\n def _get_class(self, full_name, class_name):\n full_class_name = '.'.join([full_name, class_name])\n mod_name, class_name = full_class_name.rsplit('.', 1)\n module = __import__(mod_name, globals(), locals(), class_name)\n return getattr(module, class_name)\n\n def _process_manifest_pass_one(self, full_name,\n manifest=None, plugin_path=None):\n \"\"\"\n Pass one of processing the manifest data. This updates the global\n configuration and registers Python code with the URL map.\n \"\"\"\n if not manifest:\n return\n\n manifest_path = lambda *p: self._mf_path(manifest, *p)\n manifest_iteritems = lambda *p: self._mf_iteritems(manifest, *p)\n\n # Register config variables and sections\n for section, rules in manifest_iteritems('config', 'sections'):\n self.register_config_section(*(section.split('.') + [rules]))\n for section, rules in manifest_iteritems('config', 'variables'):\n self.register_config_variables(*(section.split('.') + [rules]))\n\n # Register commands\n for command in manifest_path('commands'):\n cls = self._get_class(full_name, command['class'])\n\n # FIXME: This is all a bit hacky, we probably just want to\n # kill the SYNOPSIS attribute entirely.\n if 'input' in command:\n name = url = '%s/%s' % (command['input'], command['name'])\n cls.UI_CONTEXT = command['input']\n else:\n name = command.get('name', cls.SYNOPSIS[1])\n url = command.get('url', cls.SYNOPSIS[2])\n cls.SYNOPSIS = tuple([cls.SYNOPSIS[0], name, url,\n cls.SYNOPSIS_ARGS or cls.SYNOPSIS[3]])\n\n self.register_commands(cls)\n\n # Register worker threads\n for thr in manifest_path('threads'):\n self.register_worker(self._get_class(full_name, thr))\n\n # Register mailboxes\n package = str(full_name)\n for mailbox in manifest_path('mailboxes'):\n cls = self._get_class(package, mailbox['class'])\n priority = int(mailbox['priority'])\n register_mailbox(priority, cls)\n\n def _process_manifest_pass_two(self, full_name,\n manifest=None, plugin_path=None):\n \"\"\"\n Pass two of processing the manifest data. This maps templates and\n data to API commands and links registers classes and methods as\n hooks here and there. As these things depend both on configuration\n and the URL map, this happens as a second phase.\n \"\"\"\n if not manifest:\n return\n\n manifest_path = lambda *p: self._mf_path(manifest, *p)\n manifest_iteritems = lambda *p: self._mf_iteritems(manifest, *p)\n\n # Register javascript classes\n for fn in manifest.get('code', {}).get('javascript', []):\n class_name = fn.replace('/', '.').rsplit('.', 1)[0]\n # FIXME: Is this a good idea?\n if full_name.endswith('.'+class_name):\n parent, class_name = full_name.rsplit('.', 1)\n else:\n parent = full_name\n self.register_js(parent, class_name,\n os.path.join(plugin_path, fn))\n\n # Register CSS files\n for fn in manifest.get('code', {}).get('css', []):\n file_name = fn.replace('/', '.').rsplit('.', 1)[0]\n self.register_css(full_name, file_name,\n os.path.join(plugin_path, fn))\n\n # Register web assets\n if plugin_path:\n from mailpile.urlmap import UrlMap\n um = UrlMap(session=self.session, config=self.config)\n for url, info in manifest_iteritems('routes'):\n filename = os.path.join(plugin_path, info['file'])\n\n # Short-cut for static content\n if url.startswith('/static/'):\n self.register_web_asset(full_name, url[8:], filename,\n mimetype=info.get('mimetype', None))\n continue\n\n # Finds the right command class and register asset in\n # the right place for that particular command.\n commands = []\n if (not url.startswith('/api/')) and 'api' in info:\n url = '/api/%d%s' % (info['api'], url)\n if url[-1] == '/':\n url += 'as.html'\n for method in ('GET', 'POST', 'PUT', 'UPDATE', 'DELETE'):\n try:\n commands = um.map(None, method, url, {}, {})\n break\n except UsageError:\n pass\n\n output = [o.get_render_mode()\n for o in commands if hasattr(o, 'get_render_mode')]\n output = output and output[-1] or 'html'\n if commands:\n command = commands[-1]\n tpath = command.template_path(output.split('.')[-1],\n template=output)\n self.register_web_asset(full_name,\n 'html/' + tpath,\n filename)\n else:\n print('FIXME: Un-routable URL in manifest %s' % url)\n\n # Register email content/crypto hooks\n s = self\n for which, reg in (\n ('outgoing_content', s.register_outgoing_email_content_transform),\n ('outgoing_crypto', s.register_outgoing_email_crypto_transform),\n ('incoming_crypto', s.register_incoming_email_crypto_transform),\n ('incoming_content', s.register_incoming_email_content_transform)\n ):\n for item in manifest_path('email_transforms', which):\n name = '%3.3d_%s' % (int(item.get('priority', 999)), full_name)\n reg(name, self._get_class(full_name, item['class']))\n\n # Register search keyword extractors\n s = self\n for which, reg in (\n ('meta', s.register_meta_kw_extractor),\n ('text', s.register_text_kw_extractor),\n ('data', s.register_data_kw_extractor)\n ):\n for item in manifest_path('keyword_extractors', which):\n reg('%s.%s' % (full_name, item),\n self._get_class(full_name, item))\n\n # Register contact/vcard hooks\n for which, reg in (\n ('importers', self.register_vcard_importers),\n ('exporters', self.register_contact_exporters),\n ('context', self.register_contact_context_providers)\n ):\n for item in manifest_path('contacts', which):\n reg(self._get_class(full_name, item))\n\n # Register periodic jobs\n def reg_job(info, spd, register):\n interval, cls = info['interval'], info['class']\n callback = self._get_class(full_name, cls)\n register('%s.%s/%s-%s' % (full_name, cls, spd, interval),\n interval, callback)\n for info in manifest_path('periodic_jobs', 'fast'):\n reg_job(info, 'fast', self.register_fast_periodic_job)\n for info in manifest_path('periodic_jobs', 'slow'):\n reg_job(info, 'slow', self.register_slow_periodic_job)\n\n ucfull_name = full_name.capitalize()\n for ui_type, elems in manifest.get('user_interface', {}).iteritems():\n for hook in elems:\n if 'javascript_setup' in hook:\n js = hook['javascript_setup']\n if not js.startswith('Mailpile.'):\n hook['javascript_setup'] = '%s.%s' % (ucfull_name, js)\n if 'javascript_events' in hook:\n for event, call in hook['javascript_events'].iteritems():\n if not call.startswith('Mailpile.'):\n hook['javascript_events'][event] = '%s.%s' \\\n % (ucfull_name, call)\n self.register_ui_element(ui_type, **hook)\n\n def _process_startup_hooks(self, package,\n manifest=None, plugin_path=None):\n if not manifest:\n return\n\n manifest_path = lambda *p: self._mf_path(manifest, *p)\n\n for method_name in manifest_path('lifecycle', 'startup'):\n method = self._get_method(package, method_name)\n method(self.config)\n\n def _compat_check(self, strict=True):\n if ((strict and (not self.loading_plugin and not self.builtin)) or\n self.deprecated):\n stack = inspect.stack()\n if str(stack[2][1]) == '':\n raise PluginError('Naughty plugin tried to directly access '\n 'mailpile.plugins!')\n\n where = '->'.join(['%s:%s' % ('/'.join(stack[i][1].split('/')[-2:]),\n stack[i][2])\n for i in reversed(range(2, len(stack)-1))])\n print(('FIXME: Deprecated use of %s at %s (issue #547)'\n ) % (stack[1][3], where))\n\n def _rhtf(self, kw_hash, term, function):\n if term in kw_hash:\n raise PluginError('Already registered: %s' % term)\n kw_hash[term] = function\n\n\n ##[ Pluggable configuration ]#############################################\n\n def register_config_variables(self, *args):\n self._compat_check()\n args = list(args)\n rules = args.pop(-1)\n dest = mailpile.config.defaults.CONFIG_RULES\n path = '/'.join(args)\n for arg in args:\n dest = dest[arg][-1]\n for rname, rule in rules.iteritems():\n if rname in dest:\n raise PluginError('Variable already exist: %s/%s' % (path, rname))\n else:\n dest[rname] = rule\n\n def register_config_section(self, *args):\n self._compat_check()\n args = list(args)\n rules = args.pop(-1)\n rname = args.pop(-1)\n dest = mailpile.config.defaults.CONFIG_RULES\n path = '/'.join(args)\n for arg in args:\n dest = dest[arg][-1]\n if rname in dest:\n raise PluginError('Section already exist: %s/%s' % (path, rname))\n else:\n dest[rname] = rules\n\n\n ##[ Pluggable message transformations ]###################################\n\n INCOMING_EMAIL_ENCRYPTION = {}\n INCOMING_EMAIL_CONTENT = {}\n OUTGOING_EMAIL_CONTENT = {}\n OUTGOING_EMAIL_ENCRYPTION = {}\n\n def _txf_in(self, transforms, config, msg, kwargs):\n matched = 0\n for name in sorted(transforms.keys()):\n txf = transforms[name](config)\n msg, match, cont = txf.TransformIncoming(msg, **kwargs)\n if match:\n matched += 1\n if not cont:\n break\n return msg, matched\n\n def _txf_out(self, transforms, cfg, s, r, msg, kwa):\n matched = 0\n for name in sorted(transforms.keys()):\n txf = transforms[name](cfg)\n s, r, msg, match, cont = txf.TransformOutgoing(s, r, msg, **kwa)\n if match:\n matched += 1\n if not cont:\n break\n return s, r, msg, matched\n\n def incoming_email_crypto_transform(self, cfg, msg, **kwa):\n return self._txf_in(self.INCOMING_EMAIL_ENCRYPTION, cfg, msg, kwa)\n\n def incoming_email_content_transform(self, config, msg, **kwa):\n return self._txf_in(self.INCOMING_EMAIL_CONTENT, config, msg, kwa)\n\n def outgoing_email_content_transform(self, cfg, s, r, m, **kwa):\n return self._txf_out(self.OUTGOING_EMAIL_CONTENT, cfg, s, r, m, kwa)\n\n def outgoing_email_crypto_transform(self, cfg, s, r, m, **kwa):\n return self._txf_out(self.OUTGOING_EMAIL_ENCRYPTION, cfg, s, r, m, kwa)\n\n def register_incoming_email_crypto_transform(self, name, transform):\n return self._rhtf(self.INCOMING_EMAIL_ENCRYPTION, name, transform)\n\n def register_incoming_email_content_transform(self, name, transform):\n return self._rhtf(self.INCOMING_EMAIL_CONTENT, name, transform)\n\n def register_outgoing_email_content_transform(self, name, transform):\n return self._rhtf(self.OUTGOING_EMAIL_CONTENT, name, transform)\n\n def register_outgoing_email_crypto_transform(self, name, transform):\n return self._rhtf(self.OUTGOING_EMAIL_ENCRYPTION, name, transform)\n\n\n ##[ Pluggable keyword extractors ]########################################\n\n DATA_KW_EXTRACTORS = {}\n TEXT_KW_EXTRACTORS = {}\n META_KW_EXTRACTORS = {}\n\n def register_data_kw_extractor(self, term, function):\n self._compat_check()\n return self._rhtf(self.DATA_KW_EXTRACTORS, term, function)\n\n def register_text_kw_extractor(self, term, function):\n self._compat_check()\n return self._rhtf(self.TEXT_KW_EXTRACTORS, term, function)\n\n def register_meta_kw_extractor(self, term, function):\n self._compat_check()\n return self._rhtf(self.META_KW_EXTRACTORS, term, function)\n\n def get_data_kw_extractors(self):\n self._compat_check(strict=False)\n return self.DATA_KW_EXTRACTORS.values()\n\n def get_text_kw_extractors(self):\n self._compat_check(strict=False)\n return self.TEXT_KW_EXTRACTORS.values()\n\n def get_meta_kw_extractors(self):\n self._compat_check(strict=False)\n return self.META_KW_EXTRACTORS.values()\n\n\n ##[ Pluggable search terms ]##############################################\n\n SEARCH_TERMS = {}\n\n def get_search_term(self, term, default=None):\n self._compat_check(strict=False)\n return self.SEARCH_TERMS.get(term, default)\n\n def register_search_term(self, term, function):\n self._compat_check()\n if term in self.SEARCH_TERMS:\n raise PluginError('Already registered: %s' % term)\n self.SEARCH_TERMS[term] = function\n\n\n ##[ Pluggable keyword filters ]###########################################\n\n FILTER_HOOKS_PRE = {}\n FILTER_HOOKS_POST = {}\n\n def get_filter_hooks(self, hooks):\n self._compat_check(strict=False)\n return ([self.FILTER_HOOKS_PRE[k]\n for k in sorted(self.FILTER_HOOKS_PRE.keys())]\n + hooks +\n [self.FILTER_HOOKS_POST[k]\n for k in sorted(self.FILTER_HOOKS_POST.keys())])\n\n def register_filter_hook_pre(self, name, hook):\n self._compat_check()\n self.FILTER_HOOKS_PRE[name] = hook\n\n def register_filter_hook_post(self, name, hook):\n self._compat_check()\n self.FILTER_HOOKS_POST[name] = hook\n\n\n ##[ Pluggable vcard functions ]###########################################\n\n VCARD_IMPORTERS = {}\n VCARD_EXPORTERS = {}\n VCARD_CONTEXT_PROVIDERS = {}\n\n def _reg_vcard_plugin(self, what, cfg_sect, plugin_classes, cls, dct):\n for plugin_class in plugin_classes:\n if not plugin_class.SHORT_NAME or not plugin_class.FORMAT_NAME:\n raise PluginError(\"Please set SHORT_NAME \"\n \"and FORMAT_* attributes!\")\n if not issubclass(plugin_class, cls):\n raise PluginError(\"%s must be a %s\" % (what, cls))\n if plugin_class.SHORT_NAME in dct:\n raise PluginError(\"%s for %s already registered\"\n % (what, importer.FORMAT_NAME))\n\n if plugin_class.CONFIG_RULES:\n rules = {\n 'guid': ['VCard source UID', str, ''],\n 'description': ['VCard source description', str, '']\n }\n rules.update(plugin_class.CONFIG_RULES)\n self.register_config_section(\n 'prefs', 'vcard', cfg_sect, plugin_class.SHORT_NAME,\n [plugin_class.FORMAT_DESCRIPTION, rules, []])\n\n dct[plugin_class.SHORT_NAME] = plugin_class\n\n def register_vcard_importers(self, *importers):\n self._compat_check()\n self._reg_vcard_plugin('Importer', 'importers', importers,\n mailpile.vcard.VCardImporter,\n self.VCARD_IMPORTERS)\n\n def register_contact_exporters(self, *exporters):\n self._compat_check()\n self._reg_vcard_plugin('Exporter', 'exporters', exporters,\n mailpile.vcard.VCardExporter,\n self.VCARD_EXPORTERS)\n\n def register_contact_context_providers(self, *providers):\n self._compat_check()\n self._reg_vcard_plugin('Context provider', 'context', providers,\n mailpile.vcard.VCardContextProvider,\n self.VCARD_CONTEXT_PROVIDERS)\n\n\n ##[ Pluggable cron jobs ]#################################################\n\n FAST_PERIODIC_JOBS = {}\n SLOW_PERIODIC_JOBS = {}\n\n def register_fast_periodic_job(self, name, period, callback):\n self._compat_check()\n # FIXME: complain about duplicates?\n self.FAST_PERIODIC_JOBS[name] = (period, callback)\n\n def register_slow_periodic_job(self, name, period, callback):\n self._compat_check()\n # FIXME: complain about duplicates?\n self.SLOW_PERIODIC_JOBS[name] = (period, callback)\n\n\n ##[ Pluggable background worker threads ]################################\n\n WORKERS = []\n\n def register_worker(self, thread_obj):\n self._compat_check()\n safe_assert(hasattr(thread_obj, 'start'))\n safe_assert(hasattr(thread_obj, 'quit'))\n # FIXME: complain about duplicates?\n self.WORKERS.append(thread_obj)\n\n\n ##[ Pluggable commands ]##################################################\n\n def register_commands(self, *args):\n self._compat_check()\n COMMANDS = mailpile.commands.COMMANDS\n for cls in args:\n if cls not in COMMANDS:\n COMMANDS.append(cls)\n\n\n ##[ Pluggable javascript, CSS template and static content ]###############\n\n JS_CLASSES = {}\n CSS_FILES = {}\n WEB_ASSETS = {}\n\n def register_js(self, plugin, classname, filename):\n self.JS_CLASSES['%s.%s' % (plugin, classname)] = filename\n\n def register_css(self, plugin, classname, filename):\n self.CSS_FILES['%s.%s' % (plugin, classname)] = filename\n\n def register_web_asset(self, plugin, path, filename, mimetype='text/html'):\n if path in self.WEB_ASSETS:\n raise PluginError(_('Already registered: %s') % path)\n self.WEB_ASSETS[path] = (filename, mimetype, plugin)\n\n def get_js_classes(self):\n return self.JS_CLASSES\n\n def get_css_files(self):\n return self.CSS_FILES\n\n def get_web_asset(self, path, default=None):\n return tuple(self.WEB_ASSETS.get(path, [default, None])[0:2])\n\n\n ##[ Pluggable UI elements ]###############################################\n\n # These are the elements that exist at the moment\n UI_ELEMENTS = {\n 'settings': [],\n 'activities': [],\n 'email_activities': [], # Activities on e-mails\n 'thread_activities': [], # Activities on e-mails in a thread\n 'display_modes': [],\n 'display_refiners': [],\n 'selection_actions': []\n }\n\n def register_ui_element(self, ui_type,\n context=None, name=None,\n text=None, icon=None, description=None,\n url=None, javascript_setup=None,\n javascript_events=None, **kwargs):\n name = name.replace('/', '_')\n if name not in [e.get('name') for e in self.UI_ELEMENTS[ui_type]]:\n # FIXME: Is context valid?\n info = {\n \"context\": context or [],\n \"name\": name,\n \"text\": text,\n \"icon\": icon,\n \"description\": description,\n \"javascript_setup\": javascript_setup,\n \"javascript_events\": javascript_events,\n \"url\": url\n }\n for k, v in kwargs.iteritems():\n info[k] = v\n self.UI_ELEMENTS[ui_type].append(info)\n else:\n raise ValueError('Duplicate element: %s' % name)\n\n def get_ui_elements(self, ui_type, context):\n # FIXME: This is a bit inefficient.\n # The good thing is, it maintains a stable order.\n return [elem for elem in self.UI_ELEMENTS[ui_type]\n if context in elem['context']]\n\n\n##[ Backwards compatibility ]################################################\n\n_default_pm = PluginManager(builtin=False, deprecated=True)\n\nregister_config_variables = _default_pm.register_config_variables\nregister_config_section = _default_pm.register_config_section\nregister_data_kw_extractor = _default_pm.register_data_kw_extractor\nregister_text_kw_extractor = _default_pm.register_text_kw_extractor\nregister_meta_kw_extractor = _default_pm.register_meta_kw_extractor\nget_data_kw_extractors = _default_pm.get_data_kw_extractors\nget_text_kw_extractors = _default_pm.get_text_kw_extractors\nget_meta_kw_extractors = _default_pm.get_meta_kw_extractors\nget_search_term = _default_pm.get_search_term\nregister_search_term = _default_pm.register_search_term\nfilter_hooks = _default_pm.get_filter_hooks\nregister_filter_hook_pre = _default_pm.register_filter_hook_pre\nregister_filter_hook_post = _default_pm.register_filter_hook_post\nregister_vcard_importers = _default_pm.register_vcard_importers\nregister_contact_exporters = _default_pm.register_contact_exporters\nregister_contact_context_providers = _default_pm.register_contact_context_providers\nregister_fast_periodic_job = _default_pm.register_fast_periodic_job\nregister_slow_periodic_job = _default_pm.register_slow_periodic_job\nregister_worker = _default_pm.register_worker\nregister_commands = _default_pm.register_commands\n" }, { "path": "mailpile/plugins/autotag.py", "content": "# This is the generic auto-tagging plugin.\n#\n# We feed the classifier the same terms as go into the search engine,\n# which should allow us to actually introspect a bit into the behavior\n# of the classifier.\n\nimport math\nimport time\nimport datetime\n\nimport mailpile.util\nfrom mailpile.commands import Command\nfrom mailpile.config.base import ConfigDict\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailutils.emails import Email\nfrom mailpile.plugins import PluginManager\nfrom mailpile.util import *\n\n\n_plugins = PluginManager(builtin=__file__)\n\n\n##[ Configuration ]###########################################################\n\nTAGGERS = {}\nTRAINERS = {}\n\nAUTO_TAG_DISABLED = (None, False, '', 'off', 'false', 'fancy', 'builtin')\n\nAUTO_TAG_CONFIG = {\n 'match_tag': ['Tag we are adding to automatically', str, ''],\n 'unsure_tag': ['If unsure, add to this tag', str, ''],\n 'exclude_tags': ['Tags on messages we should never match (ham)', str, []],\n 'ignore_kws': ['Ignore messages with these keywords', str, []],\n 'corpus_size': ['How many messages do we train on?', int, 1200],\n 'threshold': ['Size of the sure/unsure ranges', float, 0.1],\n 'tagger': ['Internal class name or |shell command', str, ''],\n 'trainer': ['Internal class name or |shell commant', str, '']}\n\n_plugins.register_config_section(\n 'prefs', 'autotag', [\"Auto-tagging\", AUTO_TAG_CONFIG , []])\n\n\ndef at_identify(at_config):\n return md5_hex(at_config.match_tag,\n at_config.tagger,\n at_config.trainer)[:12]\n\n\ndef autotag_configs(config):\n done = []\n for at_config in config.prefs.autotag:\n yield at_config\n done.append(at_config.match_tag)\n\n taggers = [k for k in TAGGERS.keys() if k != '_default']\n if not taggers:\n return\n\n for tid, tag_info in config.tags.iteritems():\n auto_tagging = (tag_info.auto_tag or '')\n if (tid not in done and\n auto_tagging.lower() not in AUTO_TAG_DISABLED):\n at_config = ConfigDict(_rules=AUTO_TAG_CONFIG)\n at_config.match_tag = tid\n if auto_tagging not in taggers:\n auto_tagging = taggers[0]\n at_config.tagger = auto_tagging\n at_config.trainer = auto_tagging\n yield at_config\n\n\nclass AutoTagger(object):\n def __init__(self, tagger, trainer):\n self.tagger = tagger\n self.trainer = trainer\n self.trained = False\n\n def reset(self, at_config):\n \"\"\"Reset to an untrained state\"\"\"\n self.trainer.reset(self, at_config)\n self.trained = False\n\n def learn(self, *args):\n self.trained = True\n return self.trainer.learn(self, *args)\n\n def should_tag(self, *args):\n return self.tagger.should_tag(self, *args)\n\n\ndef SaveAutoTagger(config, at_config):\n aid = at_identify(at_config)\n at = config.autotag.get(aid)\n if at and at.trained:\n config.save_pickle(at, 'pickled-autotag.%s' % aid)\n\n\ndef LoadAutoTagger(config, at_config):\n if not config.real_hasattr('autotag'):\n config.real_setattr('autotag', {})\n aid = at_identify(at_config)\n at = config.autotag.get(aid)\n if aid not in config.autotag:\n cfn = 'pickled-autotag.%s' % aid\n try:\n config.autotag[aid] = config.load_pickle(cfn)\n except (IOError, EOFError):\n tagger = at_config.tagger\n trainer = at_config.trainer\n config.autotag[aid] = AutoTagger(\n TAGGERS.get(tagger, TAGGERS['_default'])(tagger),\n TRAINERS.get(trainer, TRAINERS['_default'])(trainer))\n SaveAutoTagger(config, at_config)\n return config.autotag[aid]\n\n\n# FIXME: This is dumb\nimport mailpile.config.manager\nmailpile.config.manager.ConfigManager.load_auto_tagger = LoadAutoTagger\nmailpile.config.manager.ConfigManager.save_auto_tagger = SaveAutoTagger\n\n\n##[ Internal classes ]########################################################\n\nclass AutoTagCommand(object):\n def __init__(self, command):\n self.command = command\n\n\nclass Tagger(AutoTagCommand):\n def should_tag(self, atagger, at_config, msg, keywords):\n \"\"\"Returns (result, evidence), result =True, False or None\"\"\"\n return (False, None)\n\n\nclass Trainer(AutoTagCommand):\n def learn(self, atagger, at_config, msg, keywords, should_tag):\n \"\"\"Learn that this message should (or should not) be tagged\"\"\"\n pass\n\n def reset(self, atagger, at_config):\n \"\"\"Reset to an untrained state (called by AutoTagger.reset)\"\"\"\n pass\n\n\nTAGGERS['_default'] = Tagger\nTRAINERS['_default'] = Trainer\n\n\n##[ Commands ]################################################################\n\n\nclass AutoTagCommand(Command):\n ORDER = ('Tagging', 9)\n\n def _get_keywords(self, e):\n idx = self._idx()\n if not hasattr(self, 'rcache'):\n self.rcache = {}\n mid = e.msg_mid()\n if mid not in self.rcache:\n kws, snippet = idx.read_message(\n self.session,\n mid,\n e.get_msg_info(field=idx.MSG_ID),\n e.get_msg(),\n e.get_msg_size(),\n int(e.get_msg_info(field=idx.MSG_DATE), 36))\n self.rcache[mid] = kws\n return self.rcache[mid]\n\n\nclass Retrain(AutoTagCommand):\n SYNOPSIS = (None, 'autotag/retrain', None, '[]')\n\n def command(self):\n return self._retrain(tags=self.args)\n\n def _retrain(self, tags=None):\n \"Retrain autotaggers\"\n session, config, idx = self.session, self.session.config, self._idx()\n tags = tags or [asb.match_tag for asb in autotag_configs(config)]\n tids = [config.get_tag(t)._key for t in tags if t]\n\n session.ui.mark(_('Retraining SpamBayes autotaggers'))\n if not config.real_hasattr('autotag'):\n config.real_setattr('autotag', {})\n\n # Find all the interesting messages! We don't look in the trash,\n # but we do look at interesting spam.\n #\n # Note: By specifically stating that we DON'T want trash, we\n # disable the search engine's default result suppression\n # and guarantee these results don't corrupt the somewhat\n # lame/broken result cache.\n #\n no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')]\n interest = {}\n for ttype in ('replied', 'read', 'tagged'):\n interest[ttype] = set()\n for tag in config.get_tags(type=ttype):\n interest[ttype] |= idx.search(session,\n ['in:%s' % tag.slug] + no_trash\n ).as_set()\n session.ui.notify(_('Have %d interesting %s messages'\n ) % (len(interest[ttype]), ttype))\n\n retrained, unreadable = [], []\n count_all = 0\n for at_config in autotag_configs(config):\n at_tag = config.get_tag(at_config.match_tag)\n if at_tag and at_tag._key in tids:\n session.ui.mark('Retraining: %s' % at_tag.name)\n\n yn = [(set(), set(), 'in:%s' % at_tag.slug, True),\n (set(), set(), '-in:%s' % at_tag.slug, False)]\n\n # Get the current message sets: tagged and untagged messages\n # excluding trash.\n for tset, mset, srch, which in yn:\n mset |= idx.search(session, [srch] + no_trash).as_set()\n\n # If we have any exclude_tags, they are particularly\n # interesting, so we'll look at them first.\n interesting = []\n for etagid in at_config.exclude_tags:\n etag = config.get_tag(etagid)\n if etag._key not in interest:\n srch = ['in:%s' % etag._key] + no_trash\n interest[etag._key] = idx.search(session, srch\n ).as_set()\n interesting.append(etag._key)\n interesting.extend(['replied', 'read', 'tagged', None])\n\n # Go through the interest types in order of preference and\n # while we still lack training data, add to the training set.\n for ttype in interesting:\n for tset, mset, srch, which in yn:\n # False positives are really annoying, and generally\n # speaking any autotagged subset should be a small\n # part of the Universe. So we divide the corpus\n # budget 33% True, 67% False.\n full_size = int(at_config.corpus_size *\n (0.33 if which else 0.67))\n want = min(full_size // len(interesting),\n max(0, full_size - len(tset)))\n # Make sure we always fully utilize our budget\n if full_size > len(tset) and not ttype:\n want = full_size - len(tset)\n\n if want:\n if ttype:\n adding = sorted(list(mset & interest[ttype]))\n else:\n adding = sorted(list(mset))\n adding = set(list(reversed(adding))[:want])\n tset |= adding\n mset -= adding\n\n # Load classifier, reset\n atagger = config.load_auto_tagger(at_config)\n atagger.reset(at_config)\n for tset, mset, srch, which in yn:\n count = 0\n # We go through the list of message in order, to avoid\n # thrashing caches too badly.\n for msg_idx in sorted(list(tset)):\n try:\n e = Email(idx, msg_idx)\n count += 1\n count_all += 1\n session.ui.mark(\n _('Reading %s (%d/%d, %s=%s)'\n ) % (e.msg_mid(), count, len(tset),\n at_tag.name, which))\n atagger.learn(at_config,\n e.get_msg(),\n self._get_keywords(e),\n which)\n play_nice_with_threads()\n if mailpile.util.QUITTING:\n return self._error('Aborted')\n except (IndexError, TypeError, ValueError,\n OSError, IOError):\n if 'autotag' in session.config.sys.debug:\n import traceback\n traceback.print_exc()\n unreadable.append(msg_idx)\n session.ui.warning(\n _('Failed to process message at =%s'\n ) % (b36(msg_idx)))\n\n # We got this far without crashing, so save the result.\n config.save_auto_tagger(at_config)\n retrained.append(at_tag.name)\n\n message = _('Retrained SpamBayes auto-tagging for %s'\n ) % ', '.join(retrained)\n session.ui.mark(message)\n return self._success(message, result={\n 'retrained': retrained,\n 'unreadable': unreadable,\n 'read_messages': count_all\n })\n\n @classmethod\n def interval_retrain(cls, session):\n \"\"\"\n Retrains autotaggers\n\n Classmethod used for periodic automatic retraining\n \"\"\"\n result = cls(session)._retrain()\n if result:\n return True\n else:\n return False\n\n\n_plugins.register_config_variables('prefs', {\n 'autotag_retrain_interval': [\n _('Periodically retrain autotagger (seconds)'), int, 24*60*60]})\n\n_plugins.register_slow_periodic_job(\n 'retrain_autotag',\n 'prefs.autotag_retrain_interval',\n Retrain.interval_retrain)\n\n\nclass Classify(AutoTagCommand):\n SYNOPSIS = (None, 'autotag/classify', None, '')\n ORDER = ('Tagging', 9)\n\n def _classify(self, emails):\n session, config, idx = self.session, self.session.config, self._idx()\n results = {}\n unknown = []\n for e in emails:\n kws = self._get_keywords(e)\n result = results[e.msg_mid()] = {}\n for at_config in autotag_configs(config):\n if not at_config.match_tag:\n continue\n at_tag = config.get_tag(at_config.match_tag)\n if not at_tag and at_config.match_tag not in unknown:\n session.ui.error(_('Unknown tag: %s'\n ) % at_config.match_tag)\n unknown.append(at_config.match_tag)\n continue\n\n atagger = config.load_auto_tagger(at_config)\n if atagger.trained:\n result[at_tag._key] = result.get(at_tag._key, [])\n result[at_tag._key].append(atagger.should_tag(\n at_config, e.get_msg(), kws\n ))\n return results\n\n def command(self):\n session, config, idx = self.session, self.session.config, self._idx()\n emails = [Email(idx, mid) for mid in self._choose_messages(self.args)]\n return self._success(_('Classified %d messages') % len(emails),\n self._classify(emails))\n\n\nclass AutoTag(Classify):\n SYNOPSIS = (None, 'autotag', None, '')\n ORDER = ('Tagging', 9)\n\n def command(self):\n session, config, idx = self.session, self.session.config, self._idx()\n emails = [Email(idx, mid) for mid in self._choose_messages(self.args)]\n scores = self._classify(emails)\n tag = {}\n for mid in scores:\n for at_config in autotag_configs(config):\n at_tag = config.get_tag(at_config.match_tag)\n if not at_tag:\n continue\n\n wants = scores[mid].get(at_tag._key, [(False, )])\n want = bool([True for w in wants if w[0]])\n\n if want is True:\n if at_config.match_tag not in tag:\n tag[at_config.match_tag] = [mid]\n else:\n tag[at_config.match_tag].append(mid)\n\n elif at_config.unsure_tag and want is None:\n if at_config.unsure_tag not in tag:\n tag[at_config.unsure_tag] = [mid]\n else:\n tag[at_config.unsure_tag].append(mid)\n\n for tid in tag:\n idx.add_tag(session, tid, msg_idxs=[int(i, 36) for i in tag[tid]])\n\n return self._success(_('Auto-tagged %d messages') % len(emails), tag)\n\n\n_plugins.register_commands(Retrain, Classify, AutoTag)\n\n\n##[ Keywords ]################################################################\n\ndef filter_hook(session, msg_mid, msg, keywords, **kwargs):\n \"\"\"Classify this message.\"\"\"\n if not kwargs.get('incoming', False):\n return keywords\n\n config = session.config\n for at_config in autotag_configs(config):\n try:\n at_tag = config.get_tag(at_config.match_tag)\n atagger = config.load_auto_tagger(at_config)\n if not atagger.trained:\n continue\n want, info = atagger.should_tag(at_config, msg, keywords)\n if want is True:\n if 'autotag' in config.sys.debug:\n session.ui.debug(('Autotagging %s with %s (w=%s, i=%s)'\n ) % (msg_mid, at_tag.name, want, info))\n keywords.add('%s:in' % at_tag._key)\n elif at_config.unsure_tag and want is None:\n unsure_tag = config.get_tag(at_config.unsure_tag)\n if 'autotag' in config.sys.debug:\n session.ui.debug(('Autotagging %s with %s (w=%s, i=%s)'\n ) % (msg_mid, unsure_tag.name,\n want, info))\n keywords.add('%s:in' % unsure_tag._key)\n except (KeyError, AttributeError, ValueError):\n pass\n\n return keywords\n\n\n# We add a filter pre-hook with a high (late) priority. Late priority to\n# maximize the amount of data we are feeding to the classifier, but a\n# pre-hook so normal filter rules will override the autotagging.\n_plugins.register_filter_hook_pre('90-autotag', filter_hook)\n" }, { "path": "mailpile/plugins/autotag_sb.py", "content": "# Add SpamBayes as an option to the autotagger. We like SpamBayes.\n#\n# We feed the classifier the same terms as go into the search engine,\n# which should allow us to actually introspect a bit into the behavior\n# of the classifier.\n\nfrom mailpile.spambayes import Classifier\n\nimport mailpile.plugins.autotag\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\n\n\ndef _classifier(autotagger):\n if not hasattr(autotagger, 'spambayes'):\n autotagger.spambayes = Classifier()\n return autotagger.spambayes\n\n\nclass SpamBayesTagger(mailpile.plugins.autotag.Trainer):\n def should_tag(self, atagger, at_config, msg, keywords):\n score, evidence = _classifier(atagger).chi2_spamprob(keywords,\n evidence=True)\n if score >= 1 - at_config.threshold:\n want = True\n elif score > at_config.threshold:\n want = None\n else:\n want = False\n return (want, score)\n\n\nclass SpamBayesTrainer(mailpile.plugins.autotag.Trainer):\n def learn(self, atagger, at_config, msg, keywords, should_tag):\n _classifier(atagger).learn(keywords, should_tag)\n\n def reset(self, atagger, at_config):\n atagger.spambayes = Classifier()\n\n\nmailpile.plugins.autotag.TAGGERS['spambayes'] = SpamBayesTagger\nmailpile.plugins.autotag.TRAINERS['spambayes'] = SpamBayesTrainer\n" }, { "path": "mailpile/plugins/backups.py", "content": "from __future__ import print_function\nimport cStringIO\nimport datetime\nimport gzip\nimport json\nimport os\nimport sys\nimport time\nimport traceback\nimport urllib\nimport zipfile\n\nfrom mailpile.auth import VerifyAndStorePassphrase\nfrom mailpile.config.defaults import APPVER\nfrom mailpile.commands import Command\nfrom mailpile.crypto.streamer import EncryptingStreamer, DecryptingStreamer\nfrom mailpile.plugins import PluginManager\nfrom mailpile.plugins.core import Quit\nfrom mailpile.i18n import ActivateTranslation\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.ui import SuppressHtmlOutput\nfrom mailpile.util import *\nfrom mailpile.vfs import FilePath, vfs\n\n\n_ = lambda t: t\n_plugins = PluginManager(builtin=__file__)\n\n\ndef _gzip(filename, data):\n gzip_data = cStringIO.StringIO()\n gzip_obj = gzip.GzipFile(filename, 'w', 9, gzip_data, 0)\n gzip_obj.write(data)\n gzip_obj.close()\n return gzip_data.getvalue()\n\n\ndef _gunzip(data):\n with gzip.GzipFile('', 'rb', 0, cStringIO.StringIO(data)) as gzf:\n return gzf.read()\n\n\ndef _decrypt(data, config):\n with DecryptingStreamer(cStringIO.StringIO(data),\n mep_key=config.get_master_key()) as fd:\n data = fd.read()\n fd.verify(_raise=IOError)\n return data\n\n\nclass MakeBackup(Command):\n \"\"\"Generate an encrypted backup of Stuff\"\"\"\n SYNOPSIS = (None, 'backup', 'backup', '[download]')\n ORDER = ('Internals', 6)\n RAISES = (SuppressHtmlOutput,)\n CONFIG_REQUIRED = True\n IS_USER_ACTIVITY = False\n\n @classmethod\n def SummarizeTags(cls, config):\n # First, decide which tags to include.\n # Not all tags are interesting! Most, but not all.\n keep = {}\n suppress = {}\n for tid, tag in config.tags.iteritems():\n if tag.type in ('tag', 'group', 'attribute', 'inbox', 'drafts',\n 'sent', 'spam', 'read', 'tagged', 'fwded',\n 'replied', 'search', 'profile'):\n if tid in config.index.TAGS:\n keep[tid] = tag\n elif tag.type == 'trash':\n suppress[tid] = tag\n\n msg_idx_set = set()\n for tid in keep:\n msg_idx_set |= config.index.TAGS[tid]\n for tid in suppress:\n msg_idx_set -= config.index.TAGS.get(tid, set([]))\n\n msg_id_list = [''] * len(config.index.INDEX)\n for msgid, msg_idx in config.index.MSGIDS.iteritems():\n if msg_idx in msg_idx_set:\n msg_id_list[msg_idx] = msgid\n\n return {\n 'tags': dict((tid, list(config.index.TAGS[tid]))\n for tid in keep),\n 'msgids': msg_id_list}\n\n @classmethod\n def MakeBackupArchive(cls, config, gnupg, what=None):\n backup_date = datetime.date.today().strftime('%Y-%m-%d')\n if what:\n backup_fn = 'Mailpile_Backup_%s_%s.zip' % (\n backup_date, ','.join(what))\n else:\n backup_fn = 'Mailpile_Backup_%s.zip' % (backup_date,)\n\n # Prep archive!\n backup_data = cStringIO.StringIO()\n backup_zip = zipfile.ZipFile(backup_data, 'w', zipfile.ZIP_DEFLATED)\n backup_zip.writestr('README.txt', (('\\n'.join([\n _(\"This is a backup of Mailpile v%(ver)s keys and configuration.\"),\n '',\n ' * ' + _(\"This backup was generated on: %(date)s.\"),\n ' * ' + _(\"The contents of this file should be encrypted.\"),\n ' * ' + _(\"The entire ZIP file must be uploaded during \"\n \"restoration.\"),\n '',\n '-- ',\n '{\"backup_date\": \"%(date)s\",',\n ' \"backup_version\": 1.0,',\n ' \"mailpile_version\": \"%(ver)s\"}'\n ])) % {'ver': APPVER, 'date': backup_date}).strip())\n backup_contents = []\n\n def _add_file(realfile, zipname):\n backup_zip.write(realfile, zipname)\n backup_contents.append(zipname)\n\n # The .ZIP is unencrypted, so generated contents needs protecting\n def _encrypt_and_add_data(filename, data):\n tempfile = os.path.join(config.tempfile_dir(), filename)\n with EncryptingStreamer(config.get_master_key(),\n dir=config.tempfile_dir()) as fd:\n fd.write(data)\n fd.save(tempfile)\n _add_file(tempfile, filename)\n safe_remove(tempfile)\n\n # What has been requested?\n if what and what[0] == 'full':\n what += ['config', 'profiles', 'keys', 'gnupg', 'vcards', 'tags']\n\n # Critical: Copy the configuration and master keys\n if not what or 'config' in what:\n for fn in (config.conf_pub, config.conf_key, config.conffile):\n _add_file(fn, os.path.basename(fn))\n\n # Critical: Copy the profile VCard data\n if not what or 'profiles' in what:\n for profile in config.vcards.find_vcards([], kinds=['profile']):\n target = os.path.basename(profile.filename)\n _add_file(profile.filename, os.path.join('vcards', target))\n\n # Critical: Copy all the private GnuPG keys!\n if not what or 'keys' in what:\n _encrypt_and_add_data('gnupg-privkeys.asc.gze',\n _gzip('gnupg-privkeys.asc', gnupg.export_privkeys()))\n\n # Recommended: Copy all the public GnuPG keys!\n if not what or 'gnupg' in what:\n _encrypt_and_add_data('gnupg-pubkeys.asc.gze',\n _gzip('gnupg-pubkeys.asc', gnupg.export_pubkeys()))\n\n # Recommended: Copy the \"interesting\" VCards.\n if not what or 'vcards' in what:\n for vcard in config.vcards.find_vcards([],\n kinds=['individual', 'group']):\n if ((what and 'full' in what)\n or vcard.recent_history()\n or vcard.crypto_policy\n or vcard.html_policy\n or vcard.pgp_key_shared\n or vcard.pgp_key):\n target = os.path.basename(vcard.filename)\n _add_file(vcard.filename, os.path.join('vcards', target))\n\n # Optional: Backup the tag structure. This is useful if we lose the\n # metadata index, but have the original e-mails. This is DISABLED BY\n # DEFAULT because it is expensive and that may not be a real use case.\n if what and 'tags' in what:\n _encrypt_and_add_data('tags.json.gze',\n _gzip('tags.json', json.dumps(cls.SummarizeTags(config))))\n\n # Finalize archive\n backup_zip.close()\n backup_data = backup_data.getvalue()\n\n return backup_fn, backup_contents, backup_data\n\n def command(self):\n session, config = self.session, self.session.config\n html_variables = session.ui.html_variables\n\n if not (html_variables and\n session.ui.valid_csrf_token(self.data.get('csrf', [''])[0])):\n raise AccessError('Invalid CSRF token')\n\n backup_fn, backup_contents, backup_data = self.MakeBackupArchive(\n config, self._gnupg(),\n what=[a for a in self.args if a not in ('download',)])\n\n if 'download' in self.args:\n encoded_fn = urllib.quote(backup_fn.encode('utf-8'))\n request = html_variables['http_request']\n request.send_http_response(200, 'OK')\n request.send_standard_headers(mimetype='application/zip',\n header_list=[\n ('Content-Length', len(backup_data)),\n ('Content-Disposition',\n 'attachment; filename*=UTF-8\\'\\'%s' % (encoded_fn,))])\n request.wfile.write(backup_data)\n raise SuppressHtmlOutput()\n\n return self._success('Generated backup', result={\n 'filename': backup_fn,\n 'contents': backup_contents,\n 'data_b64': backup_data.encode('base64')})\n\n\nAVAILABLE_BACKUPS = {}\n\nclass RestoreBackup(Command):\n \"\"\"Bootstraup setup from a backup archive\"\"\"\n SYNOPSIS = (None, 'backup/restore', 'backup/restore', '[/path/to.zip]')\n ORDER = ('Internals', 6)\n RAISES = (UrlRedirectException,)\n CONFIG_REQUIRED = False\n HTTP_AUTH_REQUIRED = 'maybe'\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = {\n 'lang': 'Language to use in UI'}\n HTTP_POST_VARS = {\n 'restore': 'date of backup to restore',\n 'password': 'Mailpile master password',\n 'keychain': 'GnuPG keychain policy: shared*, mailpile, none',\n 'os_settings': 'OS settings policy: keep, backup*',\n 'file-data': 'file data'}\n\n def _restore_PGP_keys(self, config, backup_zip, policy):\n if policy not in ('shared', 'mailpile'):\n return\n\n if policy == 'mailpile':\n config.sys.gpg_home = config.workdir\n else:\n config.sys.gpg_home = ''\n\n for keyfile in ('gnupg-pubkeys.asc.gze', 'gnupg-privkeys.asc.gze'):\n gze = backup_zip.read(keyfile)\n print('DATA: %s' % gze)\n self._gnupg().import_keys(_gunzip(_decrypt(gze, config)))\n\n\n def _adjust_paths(self, config):\n # Go through sys.mailboxes, sources.*.mailbox:\n # - if the path is outside Workdir, does not exist, clear entry\n # - if the path is inside Workdir, does not exist, create it\n # - if the path is src:, source does not exist, clear entry\n def path_ok(mbx_path):\n if 'src:' in mbx_path.raw_fp[:5]:\n return True\n elif vfs.mailbox_type(mbx_path, config):\n return True\n elif unicode(mbx_path).startswith('/Mailpile$/'):\n config.create_local_mailstore(\n self.session, name=mbx_path.raw_fp)\n return True\n else:\n return False\n\n for i, mbx_path in config.sys.mailbox.iteritems():\n mbx_path = FilePath(mbx_path)\n if not path_ok(mbx_path):\n config.sys.mailbox[i] = '/dev/null'\n\n for i, p, src in config.get_mailboxes(with_mail_source=True,\n mail_source_locals=True):\n mbx_path = FilePath(p)\n if src.mailbox[i].local and not path_ok(mbx_path):\n src.mailbox[i].local = '!CREATE'\n\n def command(self):\n global AVAILABLE_BACKUPS\n session, config = self.session, self.session.config\n message, results = '', {}\n\n if config.prefs.gpg_recipient or os.path.exists(config.conf_key):\n raise UrlRedirectException('/' + (config.sys.http_path or ''))\n\n if 'lang' in self.data:\n ActivateTranslation(session, config, self.data['lang'][0])\n\n password = ''\n if self.args and '_method' not in self.data:\n try:\n if self.args[0] in AVAILABLE_BACKUPS:\n backup_data = AVAILABLE_BACKUPS[self.args[0]]\n self.data['restore'] = [self.args[0]]\n password = session.ui.get_password(_(\"Your password: \"))\n else:\n with open(self.args[0], 'r') as fd:\n backup_data = fd.read()\n except (IOError, OSError):\n return self._error('Failed to read: %s' % self.args[0])\n elif self.data.get('_method') == 'POST':\n if 'restore' in self.data:\n backup_data = AVAILABLE_BACKUPS[self.data['restore'][0]]\n password = self.data.get('password', [''])[0]\n else:\n backup_data = self.data.get('file-data', [None])[0]\n else:\n backup_data = None\n\n if backup_data is not None:\n try:\n if isinstance(backup_data, str):\n backup_data = cStringIO.StringIO(backup_data)\n backup_zip = zipfile.ZipFile(backup_data, 'r')\n\n # Load and validate metadata (from README.txt)\n results['metadata'] = metadata = json.loads(\n backup_zip.read('README.txt').split('-- ')[1])\n results['metadata']['contents'] = backup_zip.namelist()\n backup_date = metadata['backup_date']\n if metadata['backup_version'] != 1.0:\n raise ValueError('Unrecognized backup version')\n\n # If we get this far, the backup looks good. Restore?\n if (password and\n backup_date == self.data.get('restore', [''])[0]):\n # This should be safe: we are in the setup phase where\n # almost no background stuff is running, so it should be\n # fine to just overwrite files and reload.\n config.stop_workers()\n backup_zip.extractall(config.workdir)\n VerifyAndStorePassphrase(config, password)\n\n os_gpg_home = config.sys.gpg_home\n os_gpg_binary = config.sys.gpg_binary\n os_http_port = config.sys.http_port\n os_minfree_mb = config.sys.minfree_mb\n try:\n config.load(session)\n except IOError:\n pass\n\n B = ['backup']\n if 'keep' == self.data.get('os_settings', B)[0]:\n config.sys.gpg_home = os_gpg_home\n config.sys.gpg_binary = os_gpg_binary\n config.sys.http_port = os_http_port\n config.sys.minfree_mb = os_minfree_mb\n\n self._restore_PGP_keys(config, backup_zip,\n self.data.get('keychain', ['shared'])[0])\n\n self._adjust_paths(config)\n\n config.prepare_workers(session, daemons=True)\n message = _('Backup restored')\n results['restored'] = True\n AVAILABLE_BACKUPS = {}\n else:\n message = _('Backup validated, restoration is possible')\n AVAILABLE_BACKUPS[backup_date] = backup_data\n\n except (ValueError, KeyError, zipfile.BadZipfile, IOError):\n traceback.print_exc()\n return self._error('Incomplete, invalid or corrupt backup')\n else:\n message = _('Restore from backup')\n\n results['available'] = AVAILABLE_BACKUPS.keys()\n return self._success(message, result=results)\n\n\n_plugins.register_commands(MakeBackup, RestoreBackup)\n" }, { "path": "mailpile/plugins/compose.py", "content": "import datetime\nimport email.utils\nimport os\nimport os.path\nimport re\nimport traceback\n\nimport mailpile.security as security\nfrom mailpile.commands import Command\nfrom mailpile.crypto.state import *\nfrom mailpile.crypto.mime import EncryptionFailureError, SignatureFailureError\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.plugins import PluginManager\nfrom mailpile.mailutils import NoFromAddressError, NotEditableError\nfrom mailpile.mailutils.addresses import AddressHeaderParser\nfrom mailpile.mailutils.emails import ExtractEmailAndName, Email\nfrom mailpile.mailutils.emails import PrepareMessage, MakeMessageID\nfrom mailpile.search import MailIndex\nfrom mailpile.smtp_client import SendMail\nfrom mailpile.urlmap import UrlMap\nfrom mailpile.util import *\nfrom mailpile.vcard import AddressInfo\n\nfrom mailpile.plugins.search import Search, SearchResults, View\n\n\nGLOBAL_EDITING_LOCK = MboxRLock()\n\n_plugins = PluginManager(builtin=__file__)\n\n\nclass EditableSearchResults(SearchResults):\n def __init__(self, session, idx, new, sent, **kwargs):\n SearchResults.__init__(self, session, idx, **kwargs)\n self.new_messages = new\n self.sent_messages = sent\n if new:\n self['created'] = [m.msg_mid() for m in new]\n if sent:\n self['sent'] = [m.msg_mid() for m in new]\n self['summary'] = _('Sent: %s') % self['summary']\n\n\ndef AddComposeMethods(cls):\n class newcls(cls):\n COMMAND_CACHE_TTL = 0\n COMMAND_SECURITY = security.CC_COMPOSE_EMAIL\n\n def _create_contacts(self, emails):\n try:\n from mailpile.plugins.contacts import AddContact\n AddContact(self.session,\n arg=['=%s' % e.msg_mid() for e in emails]\n ).run(recipients=True, quietly=True, internal=True)\n except (TypeError, ValueError, IndexError):\n self._ignore_exception()\n\n def _tag_emails(self, emails, tag):\n try:\n idx = self._idx()\n idx.add_tag(self.session,\n self.session.config.get_tag_id(tag),\n msg_idxs=[e.msg_idx_pos for e in emails],\n conversation=False)\n except (TypeError, ValueError, IndexError):\n self._ignore_exception()\n\n def _untag_emails(self, emails, tag):\n try:\n idx = self._idx()\n idx.remove_tag(self.session,\n self.session.config.get_tag_id(tag),\n msg_idxs=[e.msg_idx_pos for e in emails],\n conversation=False)\n except (TypeError, ValueError, IndexError):\n self._ignore_exception()\n\n def _tagger(self, emails, untag, **kwargs):\n tag = self.session.config.get_tags(**kwargs)\n if tag and untag:\n return self._untag_emails(emails, tag[0]._key)\n elif tag:\n return self._tag_emails(emails, tag[0]._key)\n\n def _tag_blank(self, emails, untag=False):\n return self._tagger(emails, untag, type='blank')\n\n def _tag_drafts(self, emails, untag=False):\n return self._tagger(emails, untag, type='drafts')\n\n def _tag_outbox(self, emails, untag=False):\n return self._tagger(emails, untag, type='outbox')\n\n def _tag_sent(self, emails, untag=False):\n return self._tagger(emails, untag, type='sent')\n\n def _track_action(self, action_type, refs):\n session, idx = self.session, self._idx()\n for tag in session.config.get_tags(type=action_type):\n idx.add_tag(session, tag._key,\n msg_idxs=[m.msg_idx_pos for m in refs])\n\n def _actualize_ephemeral(self, ephemeral_mid):\n idx = self._idx()\n\n if isinstance(ephemeral_mid, int):\n # Not actually ephemeral, just return a normal Email\n return Email(idx, ephemeral_mid)\n\n msgid, mid = ephemeral_mid.rsplit('-', 1)\n etype, etarg, msgid = msgid.split('-', 2)\n if etarg not in ('all', 'att'):\n msgid = etarg + '-' + msgid\n msgid = '<%s>' % msgid.replace('_', '@')\n etype = etype.lower()\n\n enc_msgid = idx._encode_msg_id(msgid)\n msg_idx = idx.MSGIDS.get(enc_msgid)\n if msg_idx is not None:\n # Already actualized, just return a normal Email\n return Email(idx, msg_idx)\n\n if etype == 'forward':\n refs = [Email(idx, int(mid, 36))]\n e = Forward.CreateForward(idx, self.session, refs, msgid,\n with_atts=(etarg == 'att'))[0]\n self._track_action('fwded', refs)\n\n elif etype == 'reply':\n refs = [Email(idx, int(mid, 36))]\n e = Reply.CreateReply(idx, self.session, refs, msgid,\n reply_all=(etarg == 'all'))[0]\n self._track_action('replied', refs)\n\n else:\n e = Compose.CreateMessage(idx, self.session, msgid)[0]\n\n self._tag_blank([e])\n self.session.ui.debug('Actualized: %s' % e.msg_mid())\n\n return Email(idx, e.msg_idx_pos)\n\n return newcls\n\n\nclass CompositionCommand(AddComposeMethods(Search)):\n HTTP_QUERY_VARS = {}\n HTTP_POST_VARS = {}\n UPDATE_STRING_DATA = {\n 'mid': 'metadata-ID',\n 'subject': '..',\n 'from': '..',\n 'to': '..',\n 'cc': '..',\n 'bcc': '..',\n 'body': '..',\n 'encryption': '..',\n 'attachment': '..',\n 'attach-pgp-pubkey': '..',\n }\n\n UPDATE_HEADERS = ('Subject', 'From', 'To', 'Cc', 'Bcc', 'Encryption',\n 'Attach-PGP-Pubkey')\n\n def _new_msgid(self):\n msgid = (MakeMessageID()\n .replace('.', '-') # Dots may bother JS/CSS\n .replace('_', '-')) # We use _ to encode the @ later on\n return msgid\n\n def _get_email_updates(self, idx, create=False, noneok=False, emails=None):\n # Split the argument list into files and message IDs\n files = [f[1:].strip() for f in self.args if f.startswith('<')]\n args = [a for a in self.args if not a.startswith('<')]\n\n # Message IDs can come from post data\n for mid in self.data.get('mid', []):\n args.append('=%s' % mid)\n emails = emails or [self._actualize_ephemeral(mid) for mid in\n self._choose_messages(args, allow_ephemeral=True)]\n\n update_header_set = (set(self.data.keys()) &\n set([k.lower() for k in self.UPDATE_HEADERS]))\n updates, fofs = [], 0\n for e in (emails or (create and [None]) or []):\n # If we don't have a file, check for posted data\n if len(files) not in (0, 1, len(emails)):\n return (self._error(_('Cannot update from multiple files')),\n None)\n elif len(files) == 1:\n updates.append((e, self._read_file_or_data(files[0])))\n elif files and (len(files) == len(emails)):\n updates.append((e, self._read_file_or_data(files[fofs])))\n elif update_header_set:\n # No file name, construct an update string from the POST data.\n etree = e and e.get_message_tree() or {}\n defaults = etree.get('editing_strings', {})\n\n up = []\n for hdr in self.UPDATE_HEADERS:\n if hdr.lower() in self.data:\n data = ', '.join(self.data[hdr.lower()])\n else:\n data = defaults.get(hdr.lower(), '')\n up.append('%s: %s' % (hdr, data))\n\n # This preserves in-reply-to, references and any other\n # headers we're not usually keen on editing.\n if defaults.get('headers'):\n up.append(defaults['headers'])\n\n # This weird thing converts attachment=1234:bla.txt into a\n # dict of 1234=>bla.txt values, attachment=1234 to 1234=>None.\n # .. or just keeps all attachments if nothing is specified.\n att_keep = (dict([(ai.split(':', 1) if (':' in ai)\n else (ai, None))\n for ai in self.data.get('attachment', [])])\n if 'attachment' in self.data\n else defaults.get('attachments', {}))\n for att_id, att_fn in defaults.get('attachments',\n {}).iteritems():\n if att_id in att_keep:\n fn = att_keep[att_id] or att_fn\n up.append('Attachment-%s: %s' % (att_id, fn))\n\n updates.append((e, '\\n'.join(\n up +\n ['', '\\n'.join(self.data.get('body',\n defaults.get('body', '')))]\n )))\n elif noneok:\n updates.append((e, None))\n elif 'compose' in self.session.config.sys.debug:\n sys.stderr.write('Doing nothing with %s' % update_header_set)\n fofs += 1\n\n if 'compose' in self.session.config.sys.debug:\n for e, up in updates:\n sys.stderr.write(('compose/update: Update %s with:\\n%s\\n--\\n'\n ) % ((e and e.msg_mid() or '(new'), up))\n if not updates:\n sys.stderr.write('compose/update: No updates!\\n')\n\n return updates\n\n def _return_search_results(self, message, emails,\n expand=None, new=[], sent=[], ephemeral=False,\n error=None):\n session, idx = self.session, self._idx()\n if not ephemeral:\n session.results = [e.msg_idx_pos for e in emails]\n else:\n session.results = ephemeral\n session.displayed = EditableSearchResults(session, idx,\n new, sent,\n results=session.results,\n num=len(emails),\n emails=expand)\n if error:\n return self._error(message,\n result=session.displayed,\n info=error)\n else:\n return self._success(message, result=session.displayed)\n\n def _edit_messages(self, *args, **kwargs):\n try:\n return self._real_edit_messages(*args, **kwargs)\n except NotEditableError:\n return self._error(_('Message is not editable'))\n\n def _real_edit_messages(self, emails, new=True, tag=True, ephemeral=False):\n session, idx = self.session, self._idx()\n if (not ephemeral and\n (session.ui.edit_messages(session, emails) or not new)):\n if tag:\n self._tag_blank(emails, untag=True)\n self._tag_drafts(emails)\n self.message = _('%d message(s) edited') % len(emails)\n else:\n self.message = _('%d message(s) created') % len(emails)\n self._background_save(index=True)\n session.ui.mark(self.message)\n return self._return_search_results(self.message, emails,\n expand=emails,\n new=(new and emails),\n ephemeral=ephemeral)\n\n\nclass Draft(AddComposeMethods(View)):\n \"\"\"Edit an existing draft\"\"\"\n SYNOPSIS = ('E', 'edit', 'message/draft', '[]')\n ORDER = ('Composing', 0)\n HTTP_QUERY_VARS = {\n 'mid': 'metadata-ID'\n }\n\n def _side_effects(self, emails):\n session, idx = self.session, self._idx()\n with GLOBAL_EDITING_LOCK:\n if not emails:\n session.ui.mark(_('No messages!'))\n elif session.ui.edit_messages(session, emails):\n self._tag_blank(emails, untag=True)\n self._tag_drafts(emails)\n self._background_save(index=True)\n self.message = _('%d message(s) edited') % len(emails)\n else:\n self.message = _('%d message(s) unchanged') % len(emails)\n session.ui.mark(self.message)\n return None\n\n\nclass Compose(CompositionCommand):\n \"\"\"Create a new blank e-mail for editing\"\"\"\n SYNOPSIS = ('C', 'compose', 'message/compose', \"[ephemeral]\")\n ORDER = ('Composing', 0)\n HTTP_CALLABLE = ('POST', 'GET')\n HTTP_QUERY_VARS = dict_merge(CompositionCommand.UPDATE_STRING_DATA, {\n 'cid': 'canned response metadata-ID',\n })\n\n @classmethod\n def _get_canned(cls, idx, cid):\n try:\n return Email(idx, int(cid, 36)\n ).get_editing_strings().get('body', '')\n except (ValueError, IndexError, TypeError, OSError, IOError):\n traceback.print_exc() # FIXME, ugly\n return ''\n\n @classmethod\n def CreateMessage(cls, idx, session, msgid, cid=None, ephemeral=False):\n if not ephemeral:\n local_id, lmbox = session.config.open_local_mailbox(session)\n else:\n local_id, lmbox = -1, None\n ephemeral = ['new-E-%s-mail' % msgid[1:-1].replace('@', '_')]\n profiles = session.config.vcards.find_vcards([], kinds=['profile'])\n return (Email.Create(idx, local_id, lmbox,\n save=(not ephemeral),\n msg_text=(cid and cls._get_canned(idx, cid)\n or ''),\n msg_id=msgid,\n ephemeral_mid=ephemeral and ephemeral[0],\n use_default_from=(len(profiles) == 1)),\n ephemeral)\n\n def command(self):\n if 'mid' in self.data:\n return self._error('Please use update for editing messages')\n\n session, idx = self.session, self._idx()\n cid = self.data.get('cid', [None])[0]\n\n ephemeral = (self.args and \"ephemeral\" in self.args)\n if self.data.get('_method', 'POST') != 'POST':\n ephemeral = True\n\n email, ephemeral = self.CreateMessage(idx, session, self._new_msgid(),\n cid=cid,\n ephemeral=ephemeral)\n if not ephemeral:\n self._tag_blank([email])\n\n email_updates = self._get_email_updates(idx,\n emails=[email],\n create=True)\n update_string = email_updates and email_updates[0][1]\n if update_string:\n email.update_from_string(session, update_string)\n\n return self._edit_messages([email],\n ephemeral=ephemeral,\n new=(ephemeral or not update_string))\n\n\nclass RelativeCompose(Compose):\n _ATT_MIMETYPES = ('application/pgp-signature', )\n _TEXT_PARTTYPES = ('text', 'quote', 'pgpsignedtext', 'pgpsecuretext',\n 'pgpverifiedtext')\n\n _FW_REGEXP = re.compile(r'^(fwd|fw):.*', re.IGNORECASE)\n _RE_REGEXP = re.compile(r'^(rep|re):.*', re.IGNORECASE)\n\n @staticmethod\n def prefix_subject(subject, prefix, prefix_regex):\n \"\"\"Avoids stacking several consecutive Fw: Re: Re: Re:\"\"\"\n if subject is None:\n return prefix\n elif prefix_regex.match(subject):\n return subject\n else:\n return '%s %s' % (prefix, subject)\n\n\nclass Reply(RelativeCompose):\n \"\"\"Create reply(-all) drafts to one or more messages\"\"\"\n SYNOPSIS = ('r', 'reply', 'message/reply', '[all|ephemeral] ')\n ORDER = ('Composing', 3)\n HTTP_QUERY_VARS = {\n 'mid': 'metadata-ID',\n 'cid': 'canned response metadata-ID',\n 'reply_all': 'reply to all',\n 'ephemeral': 'ephemerality',\n }\n HTTP_POST_VARS = {}\n\n @classmethod\n def _add_gpg_key(cls, idx, session, addr):\n fe, fn = ExtractEmailAndName(addr)\n vcard = session.config.vcards.get_vcard(fe)\n if vcard:\n keys = vcard.get_all('KEY')\n if keys:\n mime, fp = keys[0].value.split('data:')[1].split(',', 1)\n return \"%s <%s#%s>\" % (fn, fe, fp)\n return \"%s <%s>\" % (fn, fe)\n\n @classmethod\n def _create_from_to_cc(cls, idx, session, trees):\n config = session.config\n ahp = AddressHeaderParser()\n ref_from, ref_to, ref_cc = [], [], []\n result = {'from': '', 'to': [], 'cc': []}\n\n def merge_contact(ai):\n vcard = config.vcards.get_vcard(ai.address)\n if vcard:\n ai.merge_vcard(vcard)\n return ai\n\n # Parse the headers, so we know what we're working with. We prune\n # some of the duplicates at this stage.\n for addrs in [t['addresses'] for t in trees]:\n alist = []\n for dst, addresses in (\n (ref_from, addrs.get('reply-to') or addrs.get('from', [])),\n (ref_to, addrs.get('to', [])),\n (ref_cc, addrs.get('cc', []))):\n alist += [d.address for d in dst]\n dst.extend([a for a in addresses if a.address not in alist])\n\n # 1st, choose a from address.\n from_ai = config.vcards.choose_from_address(\n config, ref_from, ref_to, ref_cc) # Note: order matters!\n if from_ai:\n result['from'] = ahp.normalized(addresses=[from_ai],\n force_name=True)\n\n def addresses(addrs, exclude=[]):\n alist = [from_ai.address] if (from_ai) else []\n alist += [a.address for a in exclude]\n return [merge_contact(a) for a in addrs\n if a.address not in alist\n and not a.address.startswith('noreply@')\n and '@noreply' not in a.address]\n\n # If only replying to messages sent from chosen from, then this is\n # a follow-up or clarification, so just use the same headers.\n if (from_ai and\n len([e for e in ref_from\n if e and e.address == from_ai.address]) == len(ref_from)):\n if ref_to:\n result['to'] = addresses(ref_to)\n if ref_cc:\n result['cc'] = addresses(ref_cc)\n\n # Else, if replying to other people:\n # - Construct To from the From lines, excluding own from\n # - Construct Cc from the To and CC lines, except new To/From\n else:\n result['to'] = addresses(ref_from)\n result['cc'] = addresses(ref_to + ref_cc, exclude=ref_from)\n\n return result\n\n @classmethod\n def CreateReply(cls, idx, session, refs, msgid,\n reply_all=False, cid=None, ephemeral=False):\n trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True)\n for m in refs]\n\n headers = cls._create_from_to_cc(idx, session, trees)\n if not reply_all and 'cc' in headers:\n del headers['cc']\n\n ref_ids = [t['headers_lc'].get('message-id') for t in trees]\n ref_subjs = [(t['summary'][4] or t['headers_lc'].get('subject'))\n for t in trees]\n msg_bodies = []\n for t in trees:\n # FIXME: Templates/settings for how we quote replies?\n quoted = ''.join([p['data'] for p in t['text_parts']\n if p['type'] in cls._TEXT_PARTTYPES\n and p['data']])\n if quoted:\n target_width = session.config.prefs.line_length\n if target_width > 40:\n quoted = reflow_text(quoted, target_width=target_width-2)\n text = ((_('%s wrote:') % t['headers_lc']['from']) + '\\n' +\n quoted)\n msg_bodies.append('\\n\\n' + text.replace('\\n', '\\n> '))\n\n if not ephemeral:\n local_id, lmbox = session.config.open_local_mailbox(session)\n else:\n local_id, lmbox = -1, None\n fmt = 'reply-all-%s-%s' if reply_all else 'reply-%s-%s'\n ephemeral = [fmt % (msgid[1:-1].replace('@', '_'),\n refs[0].msg_mid())]\n\n if 'cc' in headers:\n fmt = _('Composing a reply from %(from)s to %(to)s, cc %(cc)s')\n else:\n fmt = _('Composing a reply from %(from)s to %(to)s')\n session.ui.debug(fmt % headers)\n\n extra_headers = []\n for tree in trees:\n try:\n if 'decrypted' in tree['crypto']['encryption']['status']:\n extra_headers.append(('x-mp-internal-should-encrypt', 'Y'))\n extra_headers.append(('Encryption', 'openpgp-sign-encrypt'))\n break\n except KeyError:\n pass\n\n if cid:\n # FIXME: Instead, we should use placeholders in the template\n # and insert the quoted bits in the right place (or\n # nowhere if the template doesn't want them).\n msg_bodies[:0] = [cls._get_canned(idx, cid)]\n\n email = Email.Create(idx, local_id, lmbox,\n msg_text='\\n\\n'.join(msg_bodies),\n msg_subject=cls.prefix_subject(\n ref_subjs[-1], 'Re:', cls._RE_REGEXP),\n msg_from=headers.get('from', None),\n msg_to=headers.get('to', []),\n msg_cc=headers.get('cc', []),\n msg_references=[i for i in ref_ids if i],\n msg_headers=extra_headers,\n msg_id=msgid,\n save=(not ephemeral),\n ephemeral_mid=ephemeral and ephemeral[0])\n\n\n return (email, ephemeral)\n\n def command(self):\n session, config, idx = self.session, self.session.config, self._idx()\n reply_all = False\n ephemeral = False\n args = list(self.args)\n if not args:\n args = [\"=%s\" % x for x in self.data.get('mid', [])]\n ephemeral = truthy((self.data.get('ephemeral') or [False])[0])\n reply_all = truthy((self.data.get('reply_all') or [False])[0])\n else:\n while args:\n if args[0].lower() == 'all':\n reply_all = args.pop(0) or True\n elif args[0].lower() == 'ephemeral':\n ephemeral = args.pop(0) or True\n else:\n break\n\n # Make sure GET does not change backend state, allow on CLI.\n if self.data.get('_method', 'POST') != 'POST':\n ephemeral = True\n\n refs = [Email(idx, i) for i in self._choose_messages(args)]\n if refs:\n try:\n cid = self.data.get('cid', [None])[0]\n email, ephemeral = self.CreateReply(idx, session, refs,\n self._new_msgid(),\n reply_all=reply_all,\n cid=cid,\n ephemeral=ephemeral)\n except NoFromAddressError:\n return self._error(_('You must configure a '\n 'From address first.'))\n\n if not ephemeral:\n self._track_action('replied', refs)\n self._tag_blank([email])\n\n return self._edit_messages([email], ephemeral=ephemeral)\n else:\n return self._error(_('No message found'))\n\n\nclass Forward(RelativeCompose):\n \"\"\"Create forwarding drafts of one or more messages\"\"\"\n SYNOPSIS = ('f', 'forward', 'message/forward', '[att|ephemeral] ')\n ORDER = ('Composing', 4)\n HTTP_QUERY_VARS = {\n 'mid': 'metadata-ID',\n 'cid': 'canned response metadata-ID',\n 'ephemeral': 'ephemerality',\n 'atts': 'forward attachments'\n }\n HTTP_POST_VARS = {}\n\n @classmethod\n def CreateForward(cls, idx, session, refs, msgid,\n with_atts=False, cid=None, ephemeral=False):\n trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True)\n for m in refs]\n ref_subjs = [t['headers_lc']['subject'] for t in trees]\n msg_bodies = []\n msg_atts = []\n for t in trees:\n # FIXME: Templates/settings for how we quote forwards?\n text = '-------- Original Message --------\\n'\n for h in ('Date', 'Subject', 'From', 'To'):\n v = t['headers_lc'].get(h.lower(), None)\n if v:\n text += '%s: %s\\n' % (h, v)\n text += '\\n'\n text += ''.join([p['data'] for p in t['text_parts']\n if p['type'] in cls._TEXT_PARTTYPES])\n msg_bodies.append(text)\n if with_atts:\n for att in t['attachments']:\n if att['mimetype'] not in cls._ATT_MIMETYPES:\n msg_atts.append(att['part'])\n\n if not ephemeral:\n local_id, lmbox = session.config.open_local_mailbox(session)\n else:\n local_id, lmbox = -1, None\n fmt = 'forward-att-%s-%s' if msg_atts else 'forward-%s-%s'\n ephemeral = [fmt % (msgid[1:-1].replace('@', '_'),\n refs[0].msg_mid())]\n\n if cid:\n # FIXME: Instead, we should use placeholders in the template\n # and insert the quoted bits in the right place (or\n # nowhere if the template doesn't want them).\n msg_bodies[:0] = [cls._get_canned(idx, cid)]\n\n email = Email.Create(idx, local_id, lmbox,\n msg_text='\\n\\n'.join(msg_bodies),\n msg_subject=cls.prefix_subject(\n ref_subjs[-1], 'Fwd:', cls._FW_REGEXP),\n msg_id=msgid,\n msg_atts=msg_atts,\n save=(not ephemeral),\n ephemeral_mid=ephemeral and ephemeral[0])\n\n return email, ephemeral\n\n def command(self):\n session, config, idx = self.session, self.session.config, self._idx()\n\n with_atts = False\n ephemeral = False\n args = list(self.args)\n if not args:\n args = [\"=%s\" % x for x in self.data.get('mid', [])]\n ephemeral = truthy((self.data.get('ephemeral') or [False])[0])\n with_atts = truthy((self.data.get('atts') or [False])[0])\n else:\n while args:\n if args[0].lower() == 'att':\n with_atts = args.pop(0) or True\n elif args[0].lower() == 'ephemeral':\n ephemeral = args.pop(0) or True\n else:\n break\n\n # Make sure GET does not change backend state\n if self.data.get('_method', 'POST') != 'POST':\n ephemeral = True\n\n if ephemeral and with_atts:\n raise UsageError(_('Sorry, ephemeral messages cannot have '\n 'attachments at this time.'))\n\n refs = [Email(idx, i) for i in self._choose_messages(args)]\n if refs:\n cid = self.data.get('cid', [None])[0]\n email, ephemeral = self.CreateForward(idx, session, refs,\n self._new_msgid(),\n with_atts=with_atts,\n cid=cid,\n ephemeral=ephemeral)\n\n if not ephemeral:\n self._track_action('fwded', refs)\n self._tag_blank([email])\n\n return self._edit_messages([email], ephemeral=ephemeral)\n else:\n return self._error(_('No message found'))\n\n\nclass Attach(CompositionCommand):\n \"\"\"Attach a file to a message\"\"\"\n SYNOPSIS = ('a', 'attach', 'message/attach', ' []')\n ORDER = ('Composing', 2)\n WITH_CONTEXT = (GLOBAL_EDITING_LOCK, )\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_QUERY_VARS = {}\n HTTP_POST_VARS = {\n 'mid': 'metadata-ID',\n 'name': '(ignored)',\n 'file-data': 'file data'\n }\n\n def command(self, emails=None):\n session, idx = self.session, self._idx()\n args = list(self.args)\n\n files = []\n filedata = {}\n if 'file-data' in self.data:\n count = 0\n for fd in self.data['file-data']:\n fn = (hasattr(fd, 'filename')\n and fd.filename or 'attach-%d.dat' % count)\n filedata[fn] = fd\n files.append(fn)\n count += 1\n else:\n if args:\n fb = security.forbid_command(self,\n security.CC_ACCESS_FILESYSTEM)\n if fb:\n return self._error(fb)\n while os.path.exists(args[-1]):\n files.append(args.pop(-1))\n\n if not files:\n return self._error(_('No files found'))\n\n if not emails:\n args.extend(['=%s' % mid for mid in self.data.get('mid', [])])\n emails = [self._actualize_ephemeral(i) for i in\n self._choose_messages(args, allow_ephemeral=True)]\n if not emails:\n return self._error(_('No messages selected'))\n\n updated = []\n errors = []\n def err(msg):\n errors.append(msg)\n session.ui.error(msg)\n for email in emails:\n subject = email.get_msg_info(MailIndex.MSG_SUBJECT)\n try:\n email.add_attachments(session, files, filedata=filedata)\n updated.append(email)\n except KeyboardInterrupt:\n raise\n except NotEditableError:\n err(_('Read-only message: %s') % subject)\n except:\n err(_('Error attaching to %s') % subject)\n self._ignore_exception()\n\n file_list = ', '.join(f.decode('utf-8') for f in files)\n if errors:\n self.message = _('Attached %s to %d messages, failed %d'\n ) % (file_list, len(updated), len(errors))\n else:\n self.message = _('Attached %s to %d messages'\n ) % (file_list, len(updated))\n\n if updated:\n self._background_save(index=True)\n\n session.ui.notify(self.message)\n return self._return_search_results(self.message, updated,\n expand=updated, error=errors)\n\n\nclass UnAttach(CompositionCommand):\n \"\"\"Remove an attachment from a message\"\"\"\n SYNOPSIS = (None, 'unattach', 'message/unattach', ' ')\n ORDER = ('Composing', 2)\n WITH_CONTEXT = (GLOBAL_EDITING_LOCK, )\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_QUERY_VARS = {}\n HTTP_POST_VARS = {\n 'mid': 'metadata-ID',\n 'att': 'Attachment IDs or filename'\n }\n\n def command(self, emails=None):\n session, idx = self.session, self._idx()\n args = list(self.args)\n atts = []\n\n if '--' in args:\n atts = args[args.index('--') + 1:]\n args = args[:args.index('--')]\n elif args:\n atts = [args.pop(-1)]\n atts.extend(self.data.get('att', []))\n\n if not emails:\n args.extend(['=%s' % mid for mid in self.data.get('mid', [])])\n emails = [self._actualize_ephemeral(i) for i in\n self._choose_messages(args, allow_ephemeral=True)]\n if not emails:\n return self._error(_('No messages selected'))\n\n updated = []\n errors = []\n def err(msg):\n errors.append(msg)\n session.ui.error(msg)\n\n for email in emails:\n subject = email.get_msg_info(MailIndex.MSG_SUBJECT)\n try:\n email.remove_attachments(session, *atts)\n updated.append(email)\n except KeyboardInterrupt:\n raise\n except NotEditableError:\n err(_('Read-only message: %s') % subject)\n except:\n err(_('Error removing from %s') % subject)\n self._ignore_exception()\n\n if errors:\n self.message = _('Removed %s from %d messages, failed %d'\n ) % (', '.join(atts), len(updated), len(errors))\n else:\n self.message = _('Removed %s from %d messages'\n ) % (', '.join(atts), len(updated))\n\n if updated:\n self._background_save(index=True)\n\n session.ui.notify(self.message)\n return self._return_search_results(self.message, updated,\n expand=updated, error=errors)\n\n\n\nclass Sendit(CompositionCommand):\n \"\"\"Mail/bounce a message (to someone)\"\"\"\n SYNOPSIS = (None, 'bounce', 'message/send', ' []')\n ORDER = ('Composing', 5)\n HTTP_CALLABLE = ('POST', )\n HTTP_QUERY_VARS = {}\n HTTP_POST_VARS = {\n 'mid': 'metadata-ID',\n 'to': 'recipients',\n 'from': 'sender e-mail'\n }\n\n # We set our events' source class explicitly, so subclasses don't\n # accidentally create orphaned mail tracking events.\n EVENT_SOURCE = 'mailpile.plugins.compose.Sendit'\n\n def command(self, emails=None):\n session, config, idx = self.session, self.session.config, self._idx()\n args = list(self.args)\n\n bounce_to = []\n while args and '@' in args[-1]:\n bounce_to.append(args.pop(-1))\n for rcpt in (self.data.get('to', []) +\n self.data.get('cc', []) +\n self.data.get('bcc', [])):\n bounce_to.extend(AddressHeaderParser(rcpt).addresses_list())\n\n sender = self.data.get('from', [None])[0]\n if not sender and bounce_to:\n sender = idx.config.get_profile().get('email', None)\n\n if not emails:\n args.extend(['=%s' % mid for mid in self.data.get('mid', [])])\n emails = [self._actualize_ephemeral(i) for i in\n self._choose_messages(args, allow_ephemeral=True)]\n\n # First make sure the draft tags are all gone, so other edits either\n # fail or complete while we wait for the lock.\n with GLOBAL_EDITING_LOCK:\n self._tag_drafts(emails, untag=True)\n self._tag_blank(emails, untag=True)\n\n # Process one at a time so we don't eat too much memory\n sent = []\n missing_keys = []\n locked_keys = []\n for email in emails:\n events = []\n try:\n msg_mid = email.get_msg_info(idx.MSG_MID)\n\n # This is a unique sending-ID. This goes in the public (meant\n # for debugging help) section of the event-log, so we take\n # care to not reveal details about the message or recipients.\n msg_sid = sha1b64(email.get_msg_info(idx.MSG_ID),\n *sorted(bounce_to))[:8]\n\n # We load up any incomplete events for sending this message\n # to this set of recipients. If nothing is in flight, create\n # a new event for tracking this operation.\n events = list(config.event_log.incomplete(\n source=self.EVENT_SOURCE,\n data_mid=msg_mid,\n data_sid=msg_sid))\n if not events:\n events.append(config.event_log.log(\n source=self.EVENT_SOURCE,\n flags=Event.RUNNING,\n message=_('Sending message'),\n data={'mid': msg_mid, 'sid': msg_sid}))\n\n SendMail(session, msg_mid,\n [PrepareMessage(config,\n email.get_msg(pgpmime=False),\n sender=sender,\n rcpts=(bounce_to or None),\n bounce=(True if bounce_to else False),\n events=events)])\n for ev in events:\n ev.flags = Event.COMPLETE\n config.event_log.log_event(ev)\n sent.append(email)\n\n # Encryption related failures are fatal, don't retry\n except (KeyLookupError,\n EncryptionFailureError,\n SignatureFailureError) as exc:\n message = unicode(exc)\n session.ui.warning(message)\n if hasattr(exc, 'missing_keys'):\n missing_keys.extend(exc.missing)\n if hasattr(exc, 'from_key'):\n # FIXME: We assume signature failures happen because\n # the key is locked. Are there any other reasons?\n locked_keys.append(exc.from_key)\n for ev in events:\n ev.flags = Event.COMPLETE\n ev.message = message\n config.event_log.log_event(ev)\n self._ignore_exception()\n\n # FIXME: Also fatal, when the SMTP server REJECTS the mail\n except:\n # We want to try that again!\n to = email.get_msg(pgpmime=False).get('x-mp-internal-rcpts',\n '').split(',')[0]\n if to:\n message = _('Could not send mail to %s') % to\n else:\n message = _('Could not send mail')\n for ev in events:\n ev.flags = Event.INCOMPLETE\n ev.message = message\n config.event_log.log_event(ev)\n session.ui.error(message)\n self._ignore_exception()\n\n if 'compose' in config.sys.debug:\n sys.stderr.write(('compose/Sendit: Send %s to %s (sent: %s)\\n'\n ) % (len(emails),\n bounce_to or '(header folks)', sent))\n\n if missing_keys:\n self.error_info['missing_keys'] = missing_keys\n if locked_keys:\n self.error_info['locked_keys'] = locked_keys\n if sent:\n self._tag_sent(sent)\n self._tag_outbox(sent, untag=True)\n for email in sent:\n email.reset_caches()\n idx.index_email(self.session, email)\n\n self._background_save(index=True)\n return self._return_search_results(\n _('Sent %d messages') % len(sent), sent, sent=sent)\n else:\n return self._error(_('Nothing was sent'))\n\n\nclass Update(CompositionCommand):\n \"\"\"Update message from a file or HTTP upload.\"\"\"\n SYNOPSIS = (None, 'update', 'message/update', ' <')\n ORDER = ('Composing', 1)\n WITH_CONTEXT = (GLOBAL_EDITING_LOCK, )\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_POST_VARS = dict_merge(CompositionCommand.UPDATE_STRING_DATA,\n Attach.HTTP_POST_VARS)\n\n def command(self, create=True, outbox=False):\n session, config, idx = self.session, self.session.config, self._idx()\n email_updates = self._get_email_updates(idx,\n create=create,\n noneok=outbox)\n\n if not email_updates:\n return self._error(_('Nothing to do!'))\n try:\n if (self.data.get('file-data') or [''])[0]:\n if not Attach(session, data=self.data).command(emails=emails):\n return self._error(_('Failed to attach files'))\n\n for email, update_string in email_updates:\n if not email:\n return self._error(_('Cannot find message'))\n break\n email.update_from_string(session, update_string, final=outbox)\n\n emails = [e for e, u in email_updates]\n message = _('%d message(s) updated') % len(email_updates)\n\n self._tag_blank(emails, untag=True)\n self._tag_drafts(emails, untag=outbox)\n self._tag_outbox(emails, untag=(not outbox))\n\n if outbox:\n self._create_contacts(emails)\n self._background_save(index=True)\n return self._return_search_results(message, emails,\n sent=emails)\n else:\n return self._edit_messages(emails, new=False, tag=False)\n except KeyLookupError as kle:\n return self._error(_('Missing encryption keys'),\n info={'missing_keys': kle.missing})\n except EncryptionFailureError as efe:\n # This should never happen, should have been prevented at key\n # lookup!\n return self._error(_('Could not encrypt message'),\n info={'to_keys': efe.to_keys})\n except SignatureFailureError as sfe:\n # FIXME: We assume signature failures happen because\n # the key is locked. Are there any other reasons?\n return self._error(_('Could not sign message'),\n info={'locked_keys': [sfe.from_key]})\n\n\nclass UpdateAndSendit(Update):\n \"\"\"Update message from an HTTP upload and move to outbox.\"\"\"\n SYNOPSIS = ('m', 'mail', 'message/update/send', None)\n\n def command(self, create=True, outbox=True):\n return Update.command(self, create=create, outbox=outbox)\n\n\nclass UnThread(CompositionCommand):\n \"\"\"Remove a message from a thread.\"\"\"\n SYNOPSIS = (None, 'unthread', 'message/unthread', None)\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = {\n 'mid': 'message-id'}\n HTTP_POST_VARS = {\n 'subject': 'Update the metadata subject as well'}\n\n def command(self):\n session, config, idx = self.session, self.session.config, self._idx()\n args = list(self.args)\n\n # On the CLI, anything after -- is the new metadata subject.\n if '--' in args:\n subject = ' '.join(args[(args.index('--')+1):])\n args = args[:args.index('--')]\n else:\n subject = self.data.get('subject', [None])[0]\n\n # Message IDs can come from post data\n for mid in self.data.get('mid', []):\n args.append('=%s' % mid)\n emails = [self._actualize_ephemeral(i) for i in\n self._choose_messages(args, allow_ephemeral=True)]\n\n if emails:\n if self.data.get('_method', 'POST') == 'POST':\n for email in emails:\n idx.unthread_message(email.msg_mid(), new_subject=subject)\n self._background_save(index=True)\n return self._return_search_results(\n _('Unthreaded %d messages') % len(emails), emails)\n else:\n return self._return_search_results(\n _('Unthread %d messages') % len(emails), emails)\n else:\n return self._error(_('Nothing to do!'))\n\n\nclass EmptyOutbox(Sendit):\n \"\"\"Try to empty the outbox.\"\"\"\n SYNOPSIS = (None, 'sendmail', None, None)\n IS_USER_ACTIVITY = False\n\n @classmethod\n def sendmail(cls, session):\n cls(session).run()\n\n def command(self):\n cfg, idx = self.session.config, self.session.config.index\n if not idx:\n return self._error(_('The index is not ready yet'))\n\n # Collect a list of messages from the outbox\n messages = []\n for tag in cfg.get_tags(type='outbox'):\n search = ['in:%s' % tag._key]\n for msg_idx_pos in idx.search(self.session, search,\n order='flat-index').as_set():\n messages.append('=%s' % b36(msg_idx_pos))\n\n # Messages no longer in the outbox get their events canceled...\n if cfg.event_log:\n events = cfg.event_log.incomplete(source='.plugins.compose.Sendit')\n for ev in events:\n if ('mid' in ev.data and\n ('=%s' % ev.data['mid']) not in messages):\n ev.flags = ev.COMPLETE\n ev.message = _('Sending cancelled.')\n cfg.event_log.log_event(ev)\n\n # Send all the mail!\n if messages:\n self.args = tuple(set(messages))\n return Sendit.command(self)\n else:\n return self._success(_('The outbox is empty'))\n\n\n_plugins.register_config_variables('prefs', {\n 'empty_outbox_interval': [_('Delay between attempts to send mail'),\n int, 90]\n})\n_plugins.register_slow_periodic_job('sendmail',\n 'prefs.empty_outbox_interval',\n EmptyOutbox.sendmail)\n_plugins.register_commands(Compose, Reply, Forward, # Create\n Draft, Update, Attach, UnAttach, # Manipulate\n UnThread, # ...\n Sendit, UpdateAndSendit, # Send\n EmptyOutbox) # ...\n" }, { "path": "mailpile/plugins/contacts.py", "content": "import os\nimport random\nimport time\nfrom email import encoders\n\nimport mailpile.config.defaults\nimport mailpile.security as security\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.crypto.gpgi import GnuPGBaseKeyGenerator, GnuPGKeyGenerator\nfrom mailpile.crypto.autocrypt import generate_autocrypt_setup_code\nfrom mailpile.plugins import EmailTransform, PluginManager\nfrom mailpile.commands import Command, Action\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailutils.addresses import AddressHeaderParser\nfrom mailpile.mailutils.emails import Email, ExtractEmails, ExtractEmailAndName\nfrom mailpile.security import SecurePassphraseStorage\nfrom mailpile.vcard import VCardLine, VCardStore, MailpileVCard, AddressInfo, GLOBAL_VCARD_LOCK\nfrom mailpile.util import *\n\n\n_plugins = PluginManager(builtin=__file__)\n\n\n##[ VCards ]########################################\n\nclass VCardCommand(Command):\n VCARD = \"vcard\"\n IS_USER_ACTIVITY = True\n WITH_CONTEXT = (GLOBAL_VCARD_LOCK,)\n\n class CommandResult(Command.CommandResult):\n IGNORE = ('line_id', 'pid', 'x-rank')\n\n def as_text(self):\n try:\n return self._as_text()\n except (KeyError, ValueError, IndexError, TypeError):\n return ''\n\n def _as_text(self):\n if isinstance(self.result, dict):\n co = self.command_obj\n if co.VCARD in self.result:\n return self._vcards_as_text([self.result[co.VCARD]])\n if co.VCARD + 's' in self.result:\n return self._vcards_as_text(self.result[co.VCARD + 's'])\n return Command.CommandResult.as_text(self)\n\n def _vcards_as_text(self, result):\n lines = []\n b64re = re.compile('base64,.*$')\n for card in result:\n if isinstance(card, list):\n for line in card:\n key = line.name\n data = re.sub(b64re, _('(BASE64 ENCODED DATA)'),\n unicode(line[key]))\n attrs = ', '.join([('%s=%s' % (k, v))\n for k, v in line.attrs\n if k not in ('pid',)])\n if attrs:\n attrs = ' (%s)' % attrs\n lines.append('%3.3s %-5.5s %s: %s%s'\n % (line.line_id,\n line.get('pid', ''),\n key, data, attrs))\n lines.append('')\n else:\n emails = [k['email'] for k in card['email']]\n photos = [k['photo'] for k in card.get('photo', [])]\n lines.append('%s %-32.32s %s'\n % (photos and ':)' or ' ',\n card['fn'] + (' (%s)' % card['note']\n if card.get('note') else ''),\n ', '.join(emails)))\n for key in [k['key'].split(',')[-1]\n for k in card.get('key', [])]:\n lines.append(' %-32.32s key:%s' % ('', key))\n return '\\n'.join(lines)\n\n def _form_defaults(self):\n return {'form': self.HTTP_POST_VARS}\n\n def _make_new_vcard(self, handle, name, note, kind):\n l = [VCardLine(name='fn', value=name),\n VCardLine(name='kind', value=kind)]\n if note:\n l.append(VCardLine(name='note', value=note))\n if kind in VCardStore.KINDS_PEOPLE:\n return MailpileVCard(VCardLine(name='email',\n value=handle, type='pref'), *l,\n config=self.session.config)\n else:\n return MailpileVCard(VCardLine(name='nickname', value=handle), *l,\n config=self.session.config)\n\n def _valid_vcard_handle(self, vc_handle):\n return (vc_handle and '@' in vc_handle[1:])\n\n def _pre_delete_vcard(self, vcard):\n pass\n\n def _vcard_list(self, vcards, mode='mpCard', info=None, simplify=False):\n info = info or {}\n if mode == 'lines':\n data = [x.as_lines() for x in vcards if x]\n else:\n data = [x.as_mpCard() for x in vcards if x]\n\n # Generate some helpful indexes for finding stuff\n by_email = {}\n by_rid = {}\n for count, vc in enumerate(vcards):\n by_rid[vc.random_uid] = count\n by_email[vc.email] = count\n for count, vc in enumerate(vcards):\n for vcl in vc.get_all('EMAIL'):\n if vcl.value not in by_email:\n by_email[vcl.value] = count\n\n # Simplify lists when there is only one element?\n if simplify and len(data) == 1:\n data = data[0]\n whatsit = self.VCARD\n else:\n whatsit = self.VCARD + 's'\n\n info.update({\n whatsit: data,\n \"emails\": by_email,\n \"rids\": by_rid,\n \"count\": len(vcards)\n })\n return info\n\n\nclass VCard(VCardCommand):\n \"\"\"Display a single vcard\"\"\"\n SYNOPSIS = (None, 'vcards/view', None, '')\n ORDER = ('Internals', 6)\n KIND = ''\n\n def command(self, save=True):\n self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n vcards = []\n for email in self.args:\n vcard = config.vcards.get_vcard(email)\n if vcard:\n vcards.append(vcard)\n else:\n session.ui.warning('No such %s: %s' % (self.VCARD, email))\n return self._success(_('Found %d results') % len(vcards),\n result=self._vcard_list(vcards, simplify=True))\n\n\nclass AddVCard(VCardCommand):\n \"\"\"Add one or more vcards\"\"\"\n SYNOPSIS = (None, 'vcards/add', None, '[all] OR = ')\n ORDER = ('Internals', 6)\n KIND = ''\n HTTP_CALLABLE = ('POST', 'PUT', 'GET')\n HTTP_POST_VARS = {\n 'email': 'E-mail address',\n 'name': 'Contact name',\n 'note': 'Note about contact',\n 'mid': 'Message ID'\n }\n COMMAND_SECURITY = security.CC_CHANGE_CONTACTS\n\n IGNORED_EMAILS_AND_DOMAINS = (\n 'reply.airbnb.com',\n 'notifications@github.com'\n )\n\n def _add_from_messages(self, args, add_recipients):\n pairs, idx = [], self._idx()\n for email in [Email(idx, i) for i in self._choose_messages(args)]:\n msg_info = email.get_msg_info()\n pairs.append(ExtractEmailAndName(msg_info[idx.MSG_FROM]))\n if add_recipients:\n people = (idx.expand_to_list(msg_info) +\n idx.expand_to_list(msg_info, field=idx.MSG_CC))\n for e in people:\n pair = ExtractEmailAndName(e)\n domain = pair[0].split('@')[-1]\n if (pair[0] not in self.IGNORED_EMAILS_AND_DOMAINS and\n domain not in self.IGNORED_EMAILS_AND_DOMAINS and\n 'noreply' not in pair[0]):\n pairs.append(pair)\n return [(p1, p2, '') for p1, p2 in pairs]\n\n def _sanity_check(self, kind, vcard):\n pass\n\n def _before_vcard_create(self, kind, triplets):\n return {}\n\n def _after_vcard_create(self, kind, vcard, state):\n pass\n\n def command(self, recipients=False, quietly=False, internal=False):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n args = list(self.args)\n\n if self.data.get('_method', 'not-http').upper() == 'GET':\n return self._success(_('Add contacts here!'),\n self._form_defaults())\n\n if (len(args) > 2\n and args[1] == '='\n and self._valid_vcard_handle(args[0])):\n handle = args[0]\n name = []\n note = []\n inname = True\n for v in args[2:]:\n if v.startswith('('):\n inname = False\n v = v[1:]\n if v.endswith(')'):\n v = v[:-1]\n if inname:\n name.append(v)\n else:\n note.append(v)\n\n triplets = [(args[0], ' '.join(name), ' '.join(note))]\n\n elif self.data:\n if self.data.get('email'):\n emails = self.data[\"email\"]\n names = self.data[\"name\"][:]\n names.extend(['' for i in range(len(names), len(emails))])\n notes = self.data.get(\"note\", [])[:]\n notes.extend(['' for i in range(len(notes), len(emails))])\n triplets = zip(emails, names, notes)\n elif self.data.get('mid'):\n mids = self.data.get('mid')\n triplets = self._add_from_messages(\n ['=%s' % mid.replace('=', '') for mid in mids])\n else:\n if args and args[0] == 'all':\n recipients = args.pop(0) and True\n triplets = self._add_from_messages(args, recipients)\n\n if triplets:\n vcards = []\n kind = self.KIND if not internal else 'internal'\n\n self._sanity_check(kind, triplets)\n state = self._before_vcard_create(kind, triplets)\n\n for handle, name, note in triplets:\n vcard = config.vcards.get(handle.lower())\n if vcard:\n if not quietly:\n session.ui.warning('Already exists: %s' % handle)\n if kind != 'profile' and vcard.kind != 'internal':\n continue\n\n if vcard and vcard.kind == 'internal':\n config.vcards.deindex_vcard(vcard)\n vcard.email = handle.lower()\n vcard.name = name\n vcard.note = note\n vcard.kind = kind\n else:\n vcard = self._make_new_vcard(handle.lower(), name, note,\n kind)\n self._after_vcard_create(kind, vcard, state)\n config.vcards.add_vcards(vcard)\n vcards.append(vcard)\n if state.get('save_config', False):\n self._background_save(config=True)\n else:\n return self._error('Nothing to do!')\n return self._success(_('Added %d contacts') % len(vcards),\n result=self._vcard_list(vcards, simplify=True))\n\n\nclass RemoveVCard(VCardCommand):\n \"\"\"Delete vcards\"\"\"\n SYNOPSIS = (None, 'vcards/remove', None, '')\n ORDER = ('Internals', 6)\n KIND = ''\n HTTP_CALLABLE = ('POST', 'DELETE')\n HTTP_POST_VARS = {\n 'email': 'delete by e-mail',\n 'rid': 'delete by x-mailpile-rid'\n }\n COMMAND_SECURITY = security.CC_CHANGE_CONTACTS\n\n def command(self):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n removed = []\n for handle in (list(self.args) +\n self.data.get('email', []) +\n self.data.get('rid', [])):\n vcard = config.vcards.get_vcard(handle)\n if vcard:\n self._pre_delete_vcard(vcard)\n config.vcards.del_vcards(vcard)\n removed.append(handle)\n else:\n session.ui.error(_('No such contact: %s') % handle)\n if removed:\n return self._success(_('Removed contacts: %s')\n % ', '.join(removed))\n else:\n return self._error(_('No contacts found'))\n\n\nclass VCardAddLines(VCardCommand):\n \"\"\"Add a lines to a VCard\"\"\"\n SYNOPSIS = (None,\n 'vcards/addlines', 'vcards/addlines',\n ' <[[]=]line> ...')\n ORDER = ('Internals', 6)\n KIND = ''\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_POST_VARS = {\n 'email': 'update by e-mail',\n 'rid': 'update by x-mailpile-rid',\n 'name': 'Line name',\n 'value': 'Line value',\n 'replace': 'int=replace line by number',\n 'replace_all': 'Boolean: replace all lines, or not',\n 'client': 'Source of this change'\n }\n COMMAND_SECURITY = security.CC_CHANGE_CONTACTS\n DEFAULT_REPLACE_ALL = False\n\n def _get_vcard(self, handle):\n return self.session.config.vcards.get_vcard(handle)\n\n def command(self):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n\n if self.args:\n handle, lines = self.args[0], self.args[1:]\n else:\n handle = self.data.get('rid', self.data.get('email', [None]))[0]\n if not handle:\n raise ValueError('Must set rid or email to choose VCard')\n name, value, replace, replace_all = (self.data.get(n, [None])[0]\n for n in ('name', 'value', 'replace', 'replace_all'))\n if not name or not value or ':' in name or '=' in name:\n raise ValueError('Must send a line name and line data')\n value = '%s:%s' % (name, value)\n if replace:\n value = '%d=%s' % (replace, value)\n elif truthy(replace_all, default=self.DEFAULT_REPLACE_ALL):\n value = '=' + value\n lines = [value]\n\n vcard = self._get_vcard(handle)\n if not vcard:\n return self._error('%s not found: %s' % (self.VCARD, handle))\n config.vcards.deindex_vcard(vcard)\n client = self.data.get('client', [vcard.USER_CLIENT])[0]\n try:\n for l in lines:\n lname = l.split(':', 1)[0].lower()\n if lname[0] == '=':\n l = l[1:].strip()\n lname = lname[1:]\n removing = [ex._line_id for ex in vcard.get_all(lname)]\n elif lname in MailpileVCard.MPCARD_SINGLETONS:\n removing = [ex._line_id for ex in vcard.get_all(lname)]\n else:\n removing = []\n\n if '=' in l[:5]:\n ln, l = l.split('=', 1)\n vcard.set_line(int(ln.strip()), VCardLine(l.strip()),\n client=client)\n\n else:\n if removing:\n vcard.remove(*removing)\n vcard.add(VCardLine(l), client=client)\n\n vcard.save()\n return self._success(_(\"Added %d lines\") % len(lines),\n result=self._vcard_list([vcard], simplify=True, info={\n 'updated': handle,\n 'added': len(lines)\n }))\n except KeyboardInterrupt:\n raise\n except:\n config.vcards.index_vcard(vcard)\n self._ignore_exception()\n return self._error(_('Error adding lines to %s') % handle)\n finally:\n config.vcards.index_vcard(vcard)\n\n\nclass VCardSet(VCardAddLines):\n \"\"\"Add a lines to a VCard, ensuring VCard exists\"\"\"\n SYNOPSIS = (None, 'vcards/set', 'vcards/set', ' <[[]=]line> ...')\n HTTP_POST_VARS = dict_merge(VCardAddLines.HTTP_POST_VARS, {\n 'fn': 'Name on card'\n })\n DEFAULT_REPLACE_ALL = True\n\n def _get_vcard(self, handle):\n vcard = self.session.config.vcards.get_vcard(handle)\n if not vcard:\n vcard = self._make_new_vcard(handle,\n self.data.get('fn', [handle])[0],\n None,\n self.KIND or 'individual')\n self.session.config.vcards.add_vcards(vcard)\n return vcard\n\n\nclass VCardRemoveLines(VCardCommand):\n \"\"\"Remove lines from a VCard\"\"\"\n SYNOPSIS = (None, 'vcards/rmlines', 'vcards/rmlines', ' ')\n ORDER = ('Internals', 6)\n KIND = ''\n HTTP_CALLABLE = ('POST', 'UPDATE')\n COMMAND_SECURITY = security.CC_CHANGE_CONTACTS\n HTTP_POST_VARS = {\n 'email': 'update by e-mail',\n 'rid': 'update by x-mailpile-rid',\n 'name': 'Line names',\n 'line_id': 'Line IDs'}\n\n def command(self):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n\n if self.args:\n handle, names, line_ids = self.args[0], [], []\n for arg in self.args[1:]:\n try:\n line_ids.append('%d' % int(arg))\n except ValueError:\n names.append(arg)\n else:\n handle = self.data.get('rid', self.data.get('email', [None]))[0]\n if not handle:\n raise ValueError('Must set rid or email to choose VCard')\n names = self.data.get('name', [])\n line_ids = self.data.get('line_id', [])\n if not (names or line_ids):\n raise ValueError('Must send a line name or line ID')\n\n vcard = config.vcards.get_vcard(handle)\n if not vcard:\n return self._error('%s not found: %s' % (self.VCARD, handle))\n config.vcards.deindex_vcard(vcard)\n removed = 0\n try:\n for lname in names:\n line_ids.extend(ex._line_id for ex in vcard.get_all(lname))\n removed += vcard.remove(*[int(li) for li in line_ids])\n vcard.save()\n return self._success(_(\"Removed %d lines\") % removed,\n result=self._vcard_list([vcard], simplify=True, info={\n 'updated': handle,\n 'removed': removed\n }))\n except KeyboardInterrupt:\n raise\n except:\n config.vcards.index_vcard(vcard)\n self._ignore_exception()\n return self._error(_('Error removing lines from %s') % handle)\n finally:\n config.vcards.index_vcard(vcard)\n\n\nclass ListVCards(VCardCommand):\n \"\"\"Find vcards\"\"\"\n SYNOPSIS = (None, 'vcards', None, '[--lines] []')\n ORDER = ('Internals', 6)\n KIND = ''\n HTTP_QUERY_VARS = {\n 'q': 'search terms',\n 'format': 'lines or mpCard (default)',\n 'count': 'how many to display (default=40)',\n 'offset': 'skip how many in the display (default=0)',\n }\n HTTP_CALLABLE = ('GET')\n\n def _augment_list_info(self, info):\n return info\n\n def command(self):\n session, config = self.session, self.session.config\n kinds = self.KIND and [self.KIND] or None\n args = list(self.args)\n\n if 'format' in self.data:\n fmt = self.data['format'][0]\n elif args and args[0] == '--lines':\n args.pop(0)\n fmt = 'lines'\n else:\n fmt = 'mpCard'\n\n if 'q' in self.data:\n terms = self.data['q']\n else:\n terms = args\n\n if 'count' in self.data:\n count = int(self.data['count'][0])\n else:\n count = 120\n\n if 'offset' in self.data:\n offset = int(self.data['offset'][0])\n else:\n offset = 0\n\n # If we're loading, stall a bit but then report current state\n loading, loaded = config.vcards.loading, config.vcards.loaded\n if loading:\n time.sleep(2)\n loading, loaded = config.vcards.loading, config.vcards.loaded\n\n vcards = config.vcards.find_vcards(terms, kinds=kinds)\n total = len(vcards)\n vcards = vcards[offset:offset + count]\n info = self._augment_list_info({\n 'terms': args,\n 'offset': offset,\n 'count': min(count, total),\n 'total': total,\n 'start': offset,\n 'end': offset + min(count, total - offset),\n 'loading': loading,\n 'loaded': loaded})\n return self._success(\n _(\"Listed %d/%d results\") % (min(total, count), total),\n result=self._vcard_list(vcards, mode=fmt, info=info))\n\n\ndef ContactVCard(parent):\n \"\"\"A factory for generating contact commands\"\"\"\n synopsis = [(t and t.replace('vcard', 'contact') or t)\n for t in parent.SYNOPSIS]\n synopsis[2] = synopsis[1]\n\n class ContactVCardCommand(parent):\n SYNOPSIS = tuple(synopsis)\n KIND = 'individual'\n ORDER = ('Tagging', 3)\n VCARD = \"contact\"\n\n return ContactVCardCommand\n\n\nclass Contact(ContactVCard(VCard)):\n \"\"\"View contacts\"\"\"\n SYNOPSIS = (None, 'contacts/view', 'contacts/view', '[]')\n\n def command(self, save=True):\n contact = VCard.command(self, save)\n # Tee-hee, monkeypatching results.\n contact[\"sent_messages\"] = 0\n contact[\"received_messages\"] = 0\n contact[\"last_contact_from\"] = 10000000000000\n contact[\"last_contact_to\"] = 10000000000000\n\n for email in contact[\"contact\"][\"email\"]:\n s = Action(self.session, \"search\",\n [\"in:Sent\", \"to:%s\" % (email[\"email\"])]).as_dict()\n contact[\"sent_messages\"] += s[\"result\"][\"stats\"][\"total\"]\n for mid in s[\"result\"][\"thread_ids\"]:\n msg = s[\"result\"][\"data\"][\"metadata\"][mid]\n if msg[\"timestamp\"] < contact[\"last_contact_to\"]:\n contact[\"last_contact_to\"] = msg[\"timestamp\"]\n contact[\"last_contact_to_msg_url\"] = msg[\"urls\"][\"thread\"]\n\n s = Action(self.session, \"search\",\n [\"from:%s\" % (email[\"email\"])]).as_dict()\n contact[\"received_messages\"] += s[\"result\"][\"stats\"][\"total\"]\n for mid in s[\"result\"][\"thread_ids\"]:\n msg = s[\"result\"][\"data\"][\"metadata\"][mid]\n if msg[\"timestamp\"] < contact[\"last_contact_from\"]:\n contact[\"last_contact_from\"] = msg[\"timestamp\"]\n contact[\"last_contact_from_msg_url\"\n ] = msg[\"urls\"][\"thread\"]\n\n if contact[\"last_contact_to\"] == 10000000000000:\n contact[\"last_contact_to\"] = False\n contact[\"last_contact_to_msg_url\"] = \"\"\n\n if contact[\"last_contact_from\"] == 10000000000000:\n contact[\"last_contact_from\"] = False\n contact[\"last_contact_from_msg_url\"] = \"\"\n\n return contact\n\n\nclass AddContact(ContactVCard(AddVCard)):\n \"\"\"Add contacts\"\"\"\n\n\nclass RemoveContact(ContactVCard(RemoveVCard)):\n \"\"\"Remove a contact\"\"\"\n\n\nclass ListContacts(ContactVCard(ListVCards)):\n SYNOPSIS = (None, 'contacts', 'contacts', '[--lines] []')\n \"\"\"Find contacts\"\"\"\n\n\nclass ContactSet(ContactVCard(VCardSet)):\n \"\"\"Set contact lines, ensuring contact exists\"\"\"\n\n\nclass ContactImport(Command):\n \"\"\"Import contacts\"\"\"\n SYNOPSIS = (None, 'contacts/import', 'contacts/import', '[]')\n ORDER = ('Internals', 6)\n HTTP_CALLABLE = ('GET', )\n COMMAND_SECURITY = security.CC_CHANGE_CONTACTS\n\n def command(self, format, terms=None, **kwargs):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n\n if not format in PluginManager.CONTACT_IMPORTERS.keys():\n session.ui.error(\"No such import format\")\n return False\n\n importer = PluginManager.CONTACT_IMPORTERS[format]\n\n if not all([x in kwargs.keys() for x in importer.required_parameters]):\n session.ui.error(\n _(\"Required parameter missing. Required parameters \"\n \"are: %s\") % \", \".join(importer.required_parameters))\n return False\n\n allparams = importer.required_parameters + importer.optional_parameters\n\n if not all([x in allparams for x in kwargs.keys()]):\n session.ui.error(\n _(\"Unknown parameter passed to importer. \"\n \"Provided %s; but known parameters are: %s\"\n ) % (\", \".join(kwargs), \", \".join(allparams)))\n return False\n\n imp = importer(kwargs)\n if terms:\n contacts = imp.filter_contacts(terms)\n else:\n contacts = imp.get_contacts()\n\n for importedcontact in contacts:\n # Check if contact exists. If yes, then update. Else create.\n pass\n\n\nclass ContactImporters(Command):\n \"\"\"Return a list of contact importers\"\"\"\n SYNOPSIS = (None, 'contacts/importers', 'contacts/importers', '')\n ORDER = ('Internals', 6)\n HTTP_CALLABLE = ('GET', )\n\n def command(self):\n res = []\n for iname, importer in CONTACT_IMPORTERS.iteritems():\n r = {}\n r[\"short_name\"] = iname\n r[\"format_name\"] = importer.format_name\n r[\"format_description\"] = importer.format_description\n r[\"optional_parameters\"] = importer.optional_parameters\n r[\"required_parameters\"] = importer.required_parameters\n res.append(r)\n\n return res\n\n\nclass AddressSearch(VCardCommand):\n \"\"\"Find addresses (in contacts or mail index)\"\"\"\n SYNOPSIS = (None, 'search/address', 'search/address', '[]')\n ORDER = ('Searching', 6)\n HTTP_QUERY_VARS = {\n 'q': 'search terms',\n 'count': 'number of results',\n 'offset': 'offset results',\n 'ms': 'deadline in ms'\n }\n\n def _boost_rank(self, boost, term, *matches):\n boost = 0.0\n for match in matches:\n match = match.lower()\n if term in match:\n if match.startswith(term):\n boost += boost * boost * (float(len(term)) / len(match))\n else:\n boost += boost * (float(len(term)) / len(match))\n return int(boost)\n\n def _vcard_addresses(self, cfg, terms, ignored_count, deadline):\n addresses = {}\n for vcard in cfg.vcards.find_vcards(terms,\n kinds=VCardStore.KINDS_PEOPLE):\n fn = vcard.get('fn')\n for email_vcl in vcard.get_all('email'):\n info = addresses.get(email_vcl.value) or {}\n info.update(AddressInfo(email_vcl.value, fn.value,\n vcard=vcard))\n info['rank'] = min(15, info.get('rank', 15))\n addresses[email_vcl.value] = info\n for term in terms:\n info['rank'] += self._boost_rank(5, term, fn.value,\n email_vcl.value)\n if len(addresses) and time.time() > deadline:\n break\n\n return addresses.values()\n\n def _index_addresses(self, cfg, terms, vcard_addresses, count, deadline):\n existing = dict([(k['address'].lower(), k) for k in vcard_addresses])\n index = self._idx()\n\n # Figure out which tags are invisible so we can skip messages marked\n # with those tags.\n invisible = set([t._key for t in cfg.get_tags(flag_hides=True)])\n matches = {}\n addresses = []\n\n # 1st, search the social graph for matches, give low priority.\n for frm in index.EMAILS:\n frm_lower = frm.lower()\n match = True\n for term in terms:\n if term not in frm_lower:\n match = False\n break\n if match:\n matches[frm] = matches.get(frm, 0) + 3\n if len(matches) > (count * 10):\n break\n elif len(matches) and time.time() > deadline:\n break\n\n # 2nd, go through at most the last 5000 messages in the index and\n # search for matching senders or recipients, give medium priority.\n # Note: This is more CPU intensive, so we do this last.\n if len(matches) < (count * 5):\n for msg_idx in xrange(max(0, len(index.INDEX)-5000),\n len(index.INDEX)):\n msg_info = index.get_msg_at_idx_pos(msg_idx)\n tags = set(msg_info[index.MSG_TAGS].split(','))\n match = not (tags & invisible)\n if match:\n frm = msg_info[index.MSG_FROM]\n search = (frm + ' ' + msg_info[index.MSG_SUBJECT]).lower()\n for term in terms:\n if term not in search:\n match = False\n break\n if match:\n matches[frm] = matches.get(frm, 0) + 1\n if len(matches) > (count * 5):\n break\n if len(matches) and time.time() > deadline:\n break\n\n # Assign info & scores!\n for frm in matches:\n email, fn = ExtractEmailAndName(frm)\n boost = min(10, matches[frm])\n for term in terms:\n boost += self._boost_rank(4, term, fn, email)\n\n if not email or '@' not in email:\n # FIXME: This may not be the right thing for alternate\n # message transports.\n pass\n elif email.lower() in existing:\n existing[email.lower()]['rank'] += boost\n else:\n info = AddressInfo(email, fn)\n info['rank'] = info.get('rank', 0) + boost\n existing[email.lower()] = info\n addresses.append(info)\n\n return addresses\n\n def command(self):\n session, config = self.session, self.session.config\n\n count = int(self.data.get('count', 10))\n offset = int(self.data.get('offset', 0))\n deadline = time.time() + float(self.data.get('ms', 150)) / 1000.0\n terms = []\n for q in self.data.get('q', []):\n terms.extend(q.lower().split())\n for a in self.args:\n terms.extend(a.lower().split())\n\n self.session.ui.mark('Searching VCards')\n vcard_addrs = self._vcard_addresses(config, terms, count, deadline)\n\n self.session.ui.mark('Searching Metadata')\n index_addrs = self._index_addresses(config, terms, vcard_addrs,\n count, deadline)\n\n self.session.ui.mark('Sorting')\n addresses = vcard_addrs + index_addrs\n addresses.sort(key=lambda k: -k['rank'])\n total = len(addresses)\n return self._success(_('Searched for addresses'), result={\n 'addresses': addresses[offset:min(offset+count, total)],\n 'displayed': min(count, total),\n 'total': total,\n 'offset': offset,\n 'count': count,\n 'start': offset,\n 'end': offset+count,\n })\n\n\ndef ProfileVCard(parent):\n \"\"\"A factory for generating profile commands\"\"\"\n synopsis = [(t and t.replace('vcard', 'profile') or t)\n for t in parent.SYNOPSIS]\n synopsis[2] = synopsis[1]\n\n class ProfileVCardCommand(parent):\n SYNOPSIS = tuple(synopsis)\n KIND = 'profile'\n ORDER = ('Tagging', 3)\n VCARD = \"profile\"\n\n DEFAULT_KEYTYPE = 'RSA3072'\n\n def _default_signature(self):\n return _('Sent using Mailpile, Free Software from www.mailpile.is')\n\n def _augment_list_info(self, info):\n info['default_sig'] = self._default_signature()\n return info\n\n def _yn(self, val, default='no'):\n return truthy(self.data.get(val, [default])[0])\n\n def _sendmail_command(self):\n # FIXME - figure out where sendmail is for reals\n return mailpile.config.defaults.DEFAULT_SENDMAIL\n\n def _sanity_check(self, kind, triplets):\n route_id = self.data.get('route_id', [None])[0]\n if (route_id or [k for k in self.data.keys() if k[:5] in\n ('route', 'smtp-', 'sourc', 'secur', 'local')]):\n if len(triplets) > 1 or kind != 'profile':\n raise ValueError('Can only configure detailed settings '\n 'for one profile at a time')\n\n # FIXME: Check more important invariants and raise\n\n def _configure_sending_route(self, vcard, route_id):\n # Sending route\n route = self.session.config.routes.get(route_id)\n protocol = self.data.get('route-protocol', ['none'])[0]\n if protocol == 'none':\n try:\n del self.session.config.routes[route_id]\n except (KeyError, IndexError):\n pass\n vcard.route = ''\n return\n elif protocol == 'local':\n route.password = route.username = route.host = ''\n route.name = _(\"Local mail\")\n route.command = self.data.get('route-command', [None]\n )[0] or self._sendmail_command()\n elif protocol in ('smtp', 'smtptls', 'smtpssl'):\n route.command = ''\n route.name = vcard.email\n for var in ('route-username', 'route-auth_type',\n 'route-host', 'route-port'):\n rvar = var.split('-', 1)[1]\n route[rvar] = self.data.get(var, [''])[0]\n if not self.data.get('route-username', [''])[0]:\n route['auth_type'] = ''\n if 'route-password' in self.data:\n route['password'] = self.data['route-password'][0]\n else:\n raise ValueError(_('Unhandled outgoing mail protocol: %s'\n ) % protocol)\n route.protocol = protocol\n vcard.route = route_id\n\n def _get_mail_spool(self):\n path = os.getenv('MAIL') or None\n user = os.getenv('USER')\n if user and not path:\n if os.path.exists('/var/spool/mail'):\n path = os.path.normpath('/var/spool/mail/%s' % user)\n if os.path.exists('/var/mail'):\n path = os.path.normpath('/var/mail/%s' % user)\n return path\n\n def _configure_mail_sources(self, vcard):\n config = self.session.config\n sources = [r[7:].rsplit('-', 1)[0] for r in self.data.keys()\n if r.startswith('source-') and r.endswith('-protocol')]\n for src_id in sources:\n prefix = 'source-%s-' % src_id\n protocol = self.data.get(prefix + 'protocol', ['none'])[0]\n def configure_source(source):\n source.host = ''\n source.username = ''\n source.enabled = self._yn(prefix + 'enabled')\n source.discovery.create_tag = True\n source.discovery.process_new = True\n if src_id not in vcard.sources():\n vcard.add_source(source._key)\n return source\n def make_new_source():\n # This little dance makes sure source is actually a\n # config section, not just an anonymous dict.\n if src_id not in config.sources:\n config.sources[src_id] = {}\n source = config.sources[src_id]\n source.profile = vcard.random_uid\n source.discovery.apply_tags = [vcard.tag]\n return configure_source(source)\n\n if protocol == 'none':\n pass\n\n elif protocol == 'local':\n source = configure_source(vcard.get_source_by_proto(\n 'local', create=src_id))\n\n elif protocol == 'spool':\n path = self._get_mail_spool()\n if not path:\n raise ValueError(_('Mail spool not found'))\n\n if path in config.sys.mailbox.values():\n raise ValueError(_('Already configured: %s') % path)\n else:\n mailbox_idx = config.sys.mailbox.append(path)\n\n source = configure_source(vcard.get_source_by_proto(\n 'local', create=src_id))\n src_id = source._key\n\n # We need to communicate with the source below,\n # so we save config to trigger instanciation.\n self._background_save(config=True, wait=True)\n\n inbox = [t._key for t in config.get_tags(type='inbox')]\n local_copy = self._yn(prefix + 'copy-local')\n if self._yn(prefix + 'delete-source'):\n policy = 'move'\n else:\n policy = 'read'\n\n src_obj = config.mail_sources[src_id]\n src_obj.take_over_mailbox(mailbox_idx,\n policy=policy,\n create_local=local_copy,\n apply_tags=inbox,\n save=False)\n\n elif protocol in ('imap', 'imap_ssl', 'imap_tls',\n 'pop3', 'pop3_ssl'):\n source = make_new_source()\n\n # Discovery policy\n disco = source.discovery\n if self._yn(prefix + 'index-all-mail'):\n if self._yn(prefix + 'leave-on-server'):\n disco.policy = 'sync'\n else:\n disco.policy = 'move'\n disco.local_copy = True\n disco.paths = ['/']\n else:\n disco.policy = 'ignore'\n disco.local_copy = False\n disco.paths = []\n disco.guess_tags = True\n\n # Connection settings\n for rvar in ('protocol', 'auth_type', 'username',\n 'host', 'port'):\n source[rvar] = self.data.get(prefix + rvar, [''])[0]\n if (prefix + 'password') in self.data:\n source['password'] = self.data[prefix + 'password'][0]\n if (self._yn(prefix + 'force-starttls')\n and source.protocol == 'imap'):\n source.protocol = 'imap_tls'\n username = source.username\n if '@' not in username:\n username += '@%s' % source.host\n source.name = username\n\n # We need to communicate with the source below,\n # so we save config to trigger instanciation.\n self._background_save(config=True, wait=True)\n src_obj = config.mail_sources[src_id]\n\n else:\n raise ValueError(_('Unhandled incoming mail protocol: %s'\n ) % protocol)\n\n def _new_key_created(self, event, vcard_rid, passphrase):\n config = self.session.config\n fingerprint = self._key_generator.generated_key\n if fingerprint:\n with GLOBAL_VCARD_LOCK:\n vcard = vcard_rid and config.vcards.get_vcard(vcard_rid)\n if vcard:\n vcard.pgp_key = fingerprint\n vcard.save()\n event.message = _('The PGP key for %s is ready for use.'\n ) % vcard.email\n else:\n event.message = _('PGP key generation is complete')\n\n # Record the passphrase!\n config.secrets[fingerprint] = {\n 'password': passphrase,\n 'policy': 'protect'}\n\n # FIXME: Toggle something that indicates we need a backup ASAP.\n self._background_save(config=True)\n else:\n event.message = _('PGP key generation failed!')\n event.data['keygen_failed'] = True\n\n event.flags = event.COMPLETE\n event.data['keygen_finished'] = int(time.time())\n config.event_log.log_event(event)\n\n def _create_new_key(self, vcard, keytype_arg):\n passphrase = generate_autocrypt_setup_code()\n random_uid = vcard.random_uid\n\n if keytype_arg[:3].upper() == 'RSA':\n keytype = GnuPGBaseKeyGenerator.KEYTYPE_RSA\n bits = int(keytype_arg[3:])\n elif keytype_arg.upper() in ('ECC', 'ED25519', 'CURVE25519'):\n keytype = GnuPGBaseKeyGenerator.KEYTYPE_CURVE25519\n bits = None\n else:\n raise ValueError('Unknown keytype: %s' % keytype_arg)\n\n key_args = {\n 'keytype': keytype,\n 'bits': bits,\n 'name': vcard.fn,\n 'email': vcard.email,\n 'passphrase': passphrase,\n 'comment': ''}\n event = Event(source=self,\n flags=Event.INCOMPLETE,\n data={'keygen_started': int(time.time()),\n 'profile_id': random_uid},\n private_data=key_args)\n self._key_generator = GnuPGKeyGenerator(\n # FIXME: Passphrase handling is a problem here\n GnuPG(self.session.config, event=event),\n event=event,\n variables=dict_merge(GnuPGBaseKeyGenerator.VARIABLES, key_args),\n on_complete=(random_uid,\n lambda: self._new_key_created(event, random_uid,\n passphrase)))\n self._key_generator.start()\n self.session.config.event_log.log_event(event)\n\n def _configure_security(self, vcard):\n openpgp_key = self.data.get('security-pgp-key', [''])[0]\n if openpgp_key:\n if openpgp_key.startswith('!CREATE'):\n key_type = openpgp_key[8:] or self.DEFAULT_KEYTYPE\n self._create_new_key(vcard, key_type)\n else:\n vcard.pgp_key = openpgp_key\n # FIXME: Schedule a background sync job which edits\n # the key to add this Account as a UID, if it\n\n else:\n vcard.remove_all('key')\n\n # Set the following even if we don't have a key, so they don't\n # get lost if the user edits settings while a key is being\n # generated - or if they just deselect a key temporarily.\n\n # Encryption policy rules\n outg_auto = self._yn('security-best-effort-crypto')\n outg_ac11 = self._yn('security-autocrypt-crypto')\n outg_sig = self._yn('security-always-sign')\n outg_enc = self._yn('security-always-encrypt')\n if outg_ac11:\n vcard.crypto_policy = 'autocrypt'\n elif outg_enc and outg_sig:\n vcard.crypto_policy = 'openpgp-sign-encrypt'\n elif outg_sig:\n vcard.crypto_policy = 'openpgp-sign'\n elif outg_enc:\n vcard.crypto_policy = 'openpgp-encrypt'\n elif outg_auto:\n vcard.crypto_policy = 'best-effort'\n else:\n vcard.crypto_policy = 'none'\n\n # Crypto formatting rules\n pgp_autocrypt = outg_ac11 or self._yn('security-use-autocrypt')\n pgp_publish = self._yn('security-publish-to-keyserver')\n pgp_keys = self._yn('security-attach-keys')\n pgp_inline = self._yn('security-prefer-inline')\n pgp_pgpmime = self._yn('security-prefer-pgpmime')\n pgp_obscure_meta = self._yn('security-obscure-metadata')\n pgp_hdr_enc = self._yn('security-openpgp-header-encrypt')\n pgp_hdr_sig = self._yn('security-openpgp-header-sign')\n pgp_hdr_none = self._yn('security-openpgp-header-none')\n pgp_hdr_both = pgp_hdr_enc and pgp_hdr_sig\n if pgp_hdr_both:\n pgp_hdr_enc = pgp_hdr_sig = False\n if pgp_pgpmime and pgp_inline:\n pgp_pgpmime = pgp_inline = False\n vcard.crypto_format = ''.join([\n 'openpgp_header:SE' if (pgp_hdr_both) else '',\n 'openpgp_header:S' if (pgp_hdr_sig) else '',\n 'openpgp_header:E' if (pgp_hdr_enc) else '',\n 'openpgp_header:N' if (pgp_hdr_none) else '',\n '+autocrypt' if (pgp_autocrypt) else '',\n '+send_keys' if (pgp_keys) else '',\n '+prefer_inline' if (pgp_inline) else '',\n '+pgpmime' if (pgp_pgpmime) else '',\n '+obscure_meta' if (pgp_obscure_meta) else '',\n '+publish' if (pgp_publish) else ''\n ])\n\n return ProfileVCardCommand\n\n\nclass Profile(ProfileVCard(VCard)):\n \"\"\"View profile\"\"\"\n\n\nclass AddProfile(ProfileVCard(AddVCard)):\n \"\"\"Add profiles (Accounts)\"\"\"\n HTTP_POST_VARS = dict_merge(AddVCard.HTTP_POST_VARS, {\n 'route_id': 'Route ID for sending mail',\n\n 'signature': '.signature',\n 'route-*': 'Route settings',\n 'source-*': 'Source settings',\n 'security-*': 'Security settings'\n })\n\n def _form_defaults(self):\n new_src_id = randomish_uid();\n return dict_merge(AddVCard._form_defaults(self), {\n 'new_src_id': new_src_id,\n 'signature': self._default_signature(),\n 'route-protocol': 'none',\n 'route-auth_type': 'password',\n 'source-NEW-protocol': 'none',\n 'source-NEW-auth_type': 'password',\n 'source-NEW-leave-on-server': True,\n 'source-NEW-index-all-mail': True,\n 'source-NEW-force-starttls': False,\n 'source-NEW-copy-local': True,\n 'source-NEW-delete-source': False,\n 'security-best-effort-crypto': True,\n 'security-autocrypt-crypto': False,\n 'security-always-sign': False,\n 'security-always-encrypt': False,\n 'security-use-autocrypt': True,\n 'security-attach-keys': False,\n 'security-prefer-inline': False,\n 'security-prefer-pgpmime': False,\n 'security-obscure-metadata': False,\n 'security-openpgp-header-encrypt': False,\n 'security-openpgp-header-sign': True,\n 'security-openpgp-header-none': False,\n 'security-publish-to-keyserver': False\n });\n\n def _before_vcard_create(self, kind, triplets, vcard=None):\n route_id = self.data.get('route_id',\n [vcard and vcard.route or None])[0]\n if route_id:\n if route_id not in self.session.config.routes:\n raise ValueError('Not a valid route ID: %s' % route_id)\n elif self.data.get('route-protocol', ['none'])[0] != 'none':\n route_id = self.session.config.routes.append({})\n\n return {\n 'save_config': True,\n 'route_id': route_id\n }\n\n def _update_vcard_from_post(self, vcard, state=None):\n if not state:\n # When editing, this doesn't run first, so we invoke it now.\n state = self._before_vcard_create(vcard.kind, [], vcard=vcard)\n\n vcard.signature = self.data.get('signature', [''])[0]\n vcard.email = self.data.get('email', [None])[0] or vcard.email\n vcard.fn = self.data.get('name', [None])[0] or vcard.fn\n\n if not vcard.tag:\n with self.session.config._lock:\n tags = self.session.config.tags\n vcard.tag = tags.append({\n 'name': vcard.email,\n 'slug': '%8.8x' % time.time(),\n 'type': 'profile',\n 'icon': 'icon-user',\n 'flag_msg_only': True,\n 'label': False,\n 'display': 'invisible'\n })\n from mailpile.plugins.tags import Slugify\n tags[vcard.tag].slug = Slugify(\n 'account-%s' % vcard.email, tags=self.session.config.tags)\n\n route_id = state.get('route_id', None)\n if route_id is not None:\n self._configure_sending_route(vcard, route_id)\n\n self._configure_mail_sources(vcard)\n self._configure_security(vcard)\n\n def _after_vcard_create(self, kind, vcard, state):\n self._update_vcard_from_post(vcard, state=state)\n\n\nclass EditProfile(AddProfile):\n \"\"\"Edit a profile\"\"\"\n SYNOPSIS = (None, None, 'profiles/edit', None)\n HTTP_QUERY_VARS = dict_merge(AddProfile.HTTP_QUERY_VARS, {\n 'rid': 'update by x-mailpile-rid'})\n\n def _vcard_to_post_vars(self, vcard):\n cp = vcard.crypto_policy or ''\n cf = vcard.crypto_format or ''\n vc_sig = vcard.signature\n default_sig = self._default_signature()\n pvars = {\n 'rid': vcard.random_uid,\n 'name': vcard.fn,\n 'email': vcard.email,\n 'signature': default_sig if (vc_sig is None) else vc_sig,\n 'password': '',\n 'route-protocol': 'none',\n 'route-auth_type': 'password',\n 'source-NEW-protocol': 'none',\n 'source-NEW-auth_type': 'password',\n 'security-pgp-key': vcard.pgp_key or '',\n 'security-best-effort-crypto': ('best-effort' in cp),\n 'security-autocrypt-crypto': ('autocrypt' in cp),\n 'security-use-autocrypt': ('autocrypt' in cf or 'autocrypt' in cp),\n 'security-always-sign': ('sign' in cp),\n 'security-always-encrypt': ('encrypt' in cp),\n 'security-attach-keys': ('send_keys' in cf),\n 'security-prefer-inline': ('prefer_inline' in cf),\n 'security-prefer-pgpmime': ('pgpmime' in cf),\n 'security-obscure-metadata': ('obscure_meta' in cf),\n 'security-openpgp-header-encrypt': ('openpgp_header:E' in cf or\n 'openpgp_header:SE' in cf),\n 'security-openpgp-header-sign': ('openpgp_header:S' in cf or\n 'openpgp_header:ES' in cf),\n 'security-openpgp-header-none': ('openpgp_header:N' in cf),\n 'security-publish-to-keyserver': ('publish' in cf)\n }\n route = self.session.config.routes.get(vcard.route or 'ha ha ha')\n if route:\n pvars.update({\n 'route-protocol': route.protocol,\n 'route-host': route.host,\n 'route-port': route.port,\n 'route-username': route.username,\n 'route-password': route.password,\n 'route-auth_type': route.auth_type,\n 'route-command': route.command\n })\n pvars['sources'] = vcard.sources()\n for sid in pvars['sources']:\n prefix = 'source-%s-' % sid\n source = self.session.config.sources.get(sid)\n disco = source.discovery\n info = {}\n for rvar in ('protocol', 'auth_type', 'host', 'port',\n 'username', 'password'):\n info[prefix + rvar] = source[rvar]\n dp = disco.policy\n if not info[prefix + 'auth_type']:\n info[prefix + 'auth_type'] = 'password'\n info[prefix + 'leave-on-server'] = (dp not in 'move')\n info[prefix + 'index-all-mail'] = (dp in ('move', 'sync', 'read')\n and disco.local_copy)\n info[prefix + 'enabled'] = source.enabled\n if source.protocol == 'imap_tls':\n info[prefix + 'protocol'] = 'imap'\n info[prefix + 'force-starttls'] = True\n else:\n info[prefix + 'force-starttls'] = False\n\n pvars.update(info)\n return pvars\n\n def command(self):\n idx = self._idx() # Make sure VCards are all loaded\n session, config = self.session, self.session.config\n\n # OK, fetch the VCard.\n safe_assert('rid' in self.data and len(self.data['rid']) == 1)\n vcard = config.vcards.get_vcard(self.data['rid'][0])\n safe_assert(vcard)\n\n if self.data.get('_method') == 'POST':\n self._update_vcard_from_post(vcard)\n self._background_save(config=True)\n vcard.save()\n return self._success(_('Account Updated!'),\n self._vcard_to_post_vars(vcard))\n else:\n return self._success(_('Edit Account'), dict_merge(\n self._form_defaults(), self._vcard_to_post_vars(vcard)))\n\n\nclass RemoveProfile(ProfileVCard(RemoveVCard)):\n \"\"\"Remove a profile\"\"\"\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = dict_merge(RemoveVCard.HTTP_QUERY_VARS, {\n 'rid': 'x-mailpile-rid of profile to remove'})\n HTTP_POST_VARS = {\n 'rid': 'x-mailpile-rid of profile to remove',\n 'trash-email': 'If yes, move linked e-mail to Trash',\n 'delete-keys': 'If yes, delete linked PGP keys',\n 'delete-tags': 'If yes, remove linked Tags'}\n\n def _trash_email(self, vcard):\n trashes = self.session.config.get_tags(type='trash', default=[])\n if vcard.tag and trashes:\n idx = self.session.config.index\n idx.add_tag(self.session, trashes[0]._key,\n msg_idxs=idx.TAGS.get(vcard.tag, set()),\n allow_message_id_clearing=True,\n conversation=False)\n\n def _delete_keys(self, vcard):\n if vcard.pgp_key:\n found = 0\n for vc in self.session.config.vcards.find_vcards([], kinds=['profile']):\n if vc.pgp_key == vcard.pgp_key:\n found += 1\n if found == 1:\n self._gnupg().delete_key(vcard.pgp_key)\n\n def _cleanup_tags(self, vcard, delete_tags=False):\n if vcard.tag:\n if delete_tags:\n from mailpile.plugins.tags import DeleteTag\n DeleteTag(self.session, arg=[vcard.tag]).run()\n else:\n self.session.config.tags[vcard.tag].type = 'attribute'\n self.session.config.tags[vcard.tag].display = 'invisible'\n self.session.config.tags[vcard.tag].label = True\n\n def _unique_usernames(self, vcard):\n config, usernames = self.session.config, set()\n\n if (vcard.route not in ('', None)) and config.routes[vcard.route].username:\n usernames.add(config.routes[vcard.route].username)\n for source_id in vcard.sources():\n if config.sources[source_id].username:\n usernames.add(config.sources[source_id].username)\n\n for msid, source in config.sources.iteritems():\n if (source.username in usernames) and (source.profile != vcard.random_uid):\n usernames.remove(source.username)\n for mrid, route in config.routes.iteritems():\n if (route.username in usernames) and (mrid != vcard.route):\n usernames.remove(source.username)\n\n return usernames\n\n def _delete_credentials(self, vcard):\n # Check every stored credential; if it is in use by any route or source that\n # doesn't belong to this VCard, leave intact. Otherwise, delete.\n usernames = self._unique_usernames(vcard)\n oauth_tks = self.session.config.oauth.tokens\n for username in (set(oauth_tks.keys()) & usernames):\n del oauth_tks[username]\n secrets = self.session.config.secrets\n for username in (set(secrets.keys()) & usernames):\n del secrets[username]\n\n def _delete_routes(self, vcard):\n if vcard.route not in (None, ''):\n found = 0\n for vc in self.session.config.vcards.find_vcards([], kinds=['profile']):\n if vc.route == vcard.route:\n found += 1\n if found == 1:\n self.session.config.routes[vcard.route] = {}\n\n def _delete_sources(self, vcard, delete_tags=False):\n config, sources = self.session.config, self.session.config.sources\n for source_id in vcard.sources():\n src = sources[source_id]\n tids = set()\n\n # Tell the worker to shut down, if any\n if source_id in config.mail_sources:\n config.mail_sources[source_id].quit()\n config.mail_sources[source_id].event.flags = Event.COMPLETE\n\n # Keep the reference to our local mailboxes in sys.mailbox\n for mbx_id, mbx_info in src.mailbox.iteritems():\n if mbx_info.primary_tag:\n tids.add(mbx_info.primary_tag)\n if mbx_info.local and (mbx_info.path[:1] == '@'):\n config.sys.mailbox[mbx_info.path[1:]] = mbx_info.local\n\n # Reconfigure all the tags\n from mailpile.plugins.tags import DeleteTag\n if src.discovery.parent_tag:\n tids.add(src.discovery.parent_tag)\n for tid in tids:\n if ((tid in config.tags)\n and config.tags[tid].type in ('mailbox', 'profile')):\n config.tags[tid].type = 'tag'\n if delete_tags:\n DeleteTag(self.session, arg=[tid]).run()\n\n # Nuke it!\n sources[source_id] = {\n 'name': 'Deleted source',\n 'enabled': False}\n\n def _trash_email_is_safe(self, vcard):\n if vcard:\n for src_id in vcard.sources():\n if self.session.config.sources[src_id].protocol == 'local':\n return False\n return True\n return False\n\n def command(self, *args, **kwargs):\n session, config = self.session, self.session.config\n\n if 'rid' in self.data:\n vcard = config.vcards.get_vcard(self.data['rid'][0])\n else:\n vcard = None\n\n if vcard and self.data.get('_method', 'not-http').upper() != 'GET':\n if self.data.get('trash-email', [''])[0].lower() == 'yes':\n self._trash_email(vcard)\n\n if self.data.get('delete-keys', [''])[0].lower() == 'yes':\n self._delete_keys(vcard)\n\n delete_tags = (self.data.get('delete-tags', [''])[0].lower() == 'yes')\n self._delete_credentials(vcard)\n self._delete_routes(vcard)\n self._delete_sources(vcard, delete_tags=delete_tags)\n self._cleanup_tags(vcard, delete_tags=delete_tags)\n self._background_save(config=True, index=True)\n\n return RemoveVCard.command(self, *args, **kwargs)\n\n return self._success(_(\"Remove account\"), result=dict_merge(\n self._form_defaults(), {\n 'rid': vcard.random_uid if vcard else None,\n 'trash_email_is_safe': self._trash_email_is_safe(vcard),\n 'profile': (self._vcard_list([vcard])['profiles'][0]\n if vcard else None)}))\n\n\nclass ListProfiles(ProfileVCard(ListVCards)):\n \"\"\"Find profiles\"\"\"\n SYNOPSIS = (None, 'profiles', 'profiles', '[--lines] []')\n\n\nclass ProfileSet(ProfileVCard(VCardSet)):\n \"\"\"Set contact lines, ensuring contact exists\"\"\"\n\n\nclass ChooseFromAddress(Command):\n \"\"\"Display a single vcard\"\"\"\n SYNOPSIS = (None, 'profiles/choose_from', 'profiles/choose_from',\n '')\n ORDER = ('Internals', 6)\n HTTP_CALLABLE = ('GET',)\n HTTP_QUERY_VARS = {\n 'mid': 'Message ID',\n 'email': 'E-mail address',\n 'no_from': 'Ignore From: lines'\n }\n\n def command(self):\n idx, vcards = self._idx(), self.session.config.vcards\n\n emails = [e for e in self.args if '@' in e]\n emails.extend(self.data.get('email', []))\n\n messages = self._choose_messages(\n [m for m in self.args if '@' not in m] +\n ['=%s' % mid for mid in self.data.get('mid', [])]\n )\n for msg_idx_pos in messages:\n try:\n msg_info = idx.get_msg_at_idx_pos(msg_idx_pos)\n msg_emails = (idx.expand_to_list(msg_info, field=idx.MSG_TO) +\n idx.expand_to_list(msg_info, field=idx.MSG_CC))\n emails.extend(msg_emails)\n if 'no_from' not in self.data:\n emails.append(msg_info[idx.MSG_FROM])\n except ValueError:\n pass\n\n addrs = [ai for ee in emails\n for ai in AddressHeaderParser(unicode_data=ee)]\n return self._success(_('Choosing from address'), result={\n 'emails': addrs,\n 'from': vcards.choose_from_address(self.session.config, addrs)\n })\n\n\nclass ContentTxf(EmailTransform):\n def TransformOutgoing(self, sender, rcpts, msg, **kwargs):\n txf_matched, txf_continue = False, True\n\n profile = self._get_sender_profile(sender, kwargs)\n sig = profile.get('signature')\n if sig:\n part = self._get_first_part(msg, 'text/plain')\n if part is not None:\n msg_text = (part.get_payload(decode=True) or '\\n\\n'\n ).replace('\\r', '').decode('utf-8')\n if '\\n-- \\n' not in msg_text:\n msg_text = msg_text.strip() + '\\n\\n-- \\n' + sig\n try:\n msg_text.encode('us-ascii')\n need_utf8 = False\n except (UnicodeEncodeError, UnicodeDecodeError):\n msg_text = msg_text.encode('utf-8')\n need_utf8 = True\n\n part.set_payload(msg_text)\n if need_utf8:\n part.set_charset('utf-8')\n while 'content-transfer-encoding' in part:\n del part['content-transfer-encoding']\n encoders.encode_base64(part)\n\n txf_matched = True\n\n return sender, rcpts, msg, txf_matched, txf_continue\n\n\n_plugins.register_commands(VCard, AddVCard, RemoveVCard, ListVCards,\n VCardAddLines, VCardSet, VCardRemoveLines)\n_plugins.register_commands(Contact, AddContact, RemoveContact, ListContacts,\n ContactSet, AddressSearch)\n_plugins.register_commands(Profile, AddProfile, EditProfile,\n RemoveProfile, ListProfiles, ProfileSet,\n ChooseFromAddress)\n_plugins.register_commands(ContactImport, ContactImporters)\n\n_plugins.register_outgoing_email_content_transform('100_sender_vc', ContentTxf)\n" }, { "path": "mailpile/plugins/core.py", "content": "# These are the Mailpile core commands, the public \"API\" we expose for\n# searching, tagging and editing e-mail.\n#\n# FIXME: This should probably be broken into smaller modules\n#\nimport datetime\nimport json\nimport os\nimport random\nimport re\nimport socket\nimport subprocess\nimport sys\nimport traceback\nimport thread\nimport threading\nimport time\nimport webbrowser\n\nimport mailpile.util\nimport mailpile.postinglist\nimport mailpile.security as security\nimport mailpile.platforms\nfrom mailpile.commands import *\nfrom mailpile.config.validators import WebRootCheck\nfrom mailpile.crypto.gpgi import GnuPG\nfrom mailpile.eventlog import Event\nfrom mailpile.i18n import gettext as _\nfrom mailpile.i18n import ngettext as _n\nfrom mailpile.mailboxes import IsMailbox\nfrom mailpile.mailutils.emails import ClearParseCache, Email\nfrom mailpile.postinglist import GlobalPostingList\nfrom mailpile.plugins import PluginManager\nfrom mailpile.safe_popen import MakePopenUnsafe, MakePopenSafe\nfrom mailpile.search import MailIndex\nfrom mailpile.util import *\nfrom mailpile.vcard import AddressInfo\nfrom mailpile.vfs import vfs, FilePath\n\n_plugins = PluginManager(builtin=__file__)\n\n\nclass Load(Command):\n \"\"\"Load or reload the metadata index\"\"\"\n SYNOPSIS = (None, 'load', None, None)\n ORDER = ('Internals', 1)\n CONFIG_REQUIRED = False\n IS_INTERACTIVE = True\n\n def command(self, reset=True, wait=True, wait_all=False, quiet=False):\n try:\n if self._idx(reset=reset,\n wait=wait,\n wait_all=wait_all,\n quiet=quiet):\n return self._success(_('Loaded metadata index'))\n else:\n return self._error(_('Failed to load metadata index'))\n except IOError:\n return self._error(_('Failed to decrypt configuration, '\n 'please log in!'))\n\n\nclass Rescan(Command):\n \"\"\"Add new messages to index\"\"\"\n SYNOPSIS = (None, 'rescan', 'rescan',\n '[full|vcards|vcards:|sources|mailboxes|both|mailbox:|]')\n ORDER = ('Internals', 2)\n LOG_PROGRESS = True\n\n HTTP_CALLABLE = ('POST',)\n HTTP_POST_VARS = {\n 'which': '[full|vcards|vcards:|both|mailboxes|sources|]'\n }\n\n def _progress(self, progress):\n self.session.ui.mark(progress)\n self.event.data[\"progress\"].append((int(time.time()), progress))\n\n def command(self, slowly=False, cron=False):\n session, config, idx = self.session, self.session.config, self._idx()\n self.event.data[\"progress\"] = []\n args = list(self.args)\n if 'which' in self.data:\n args.extend(self.data['which'])\n\n # Abort if we are out of disk space\n full_path = config.need_more_disk_space()\n if full_path:\n return self._error(_('Insufficient free space in %s') % full_path)\n\n # Pretend we're idle, to make rescan go fast fast.\n if not slowly:\n mailpile.util.LAST_USER_ACTIVITY = 0\n\n # Cron always runs the rescan command, no matter what else\n if cron:\n self._run_rescan_command(session)\n\n a0lower = (args and args[0] or '').lower()\n if a0lower.startswith('vcards'):\n return self._success(_('Rescanned vCards'),\n result=self._rescan_vcards(session, args[0]))\n elif (a0lower in ('both', 'mailboxes', 'sources', 'editable')\n or a0lower.startswith('mailbox:')):\n return self._success(_('Rescanned mailboxes'),\n result=self._rescan_mailboxes(session,\n which=a0lower))\n elif a0lower == 'full':\n config.flush_mbox_cache(session, wait=True)\n args.pop(0)\n\n # Clear the cache first, in case the user is flailing about\n ClearParseCache(full=True)\n\n msg_idxs = self._choose_messages(args)\n if msg_idxs:\n for msg_idx_pos in msg_idxs:\n e = Email(idx, msg_idx_pos)\n try:\n self._progress('Re-indexing %s' % e.msg_mid())\n idx.index_email(self.session, e)\n except KeyboardInterrupt:\n self._progress('Interrupted')\n raise\n except:\n self._ignore_exception()\n session.ui.warning(_('Failed to reindex: %s'\n ) % e.msg_mid())\n\n self.event.data[\"messages\"] = len(msg_idxs)\n self.session.config.event_log.log_event(self.event)\n self._background_save(index=True)\n\n return self._success(_('Indexed %d messages') % len(msg_idxs),\n result={'messages': len(msg_idxs)})\n\n else:\n deadline = (int(time.time() + 0.75 * config.prefs.rescan_interval)\n if cron else None)\n if 'rescan' in config._running:\n return self._success(_('Rescan already in progress'))\n config._running['rescan'] = True\n try:\n results = {}\n results.update(self._rescan_vcards(session, 'vcards'))\n results.update(self._rescan_mailboxes(session,\n deadline=deadline,\n which=('mailboxes' if cron else 'both'),\n force=(not cron and not slowly)))\n\n self.event.data.update(results)\n self.session.config.event_log.log_event(self.event)\n if 'aborted' in results:\n self._progress('Aborted')\n raise KeyboardInterrupt()\n return self._success(_('Rescanned vCards and mailboxes'),\n result=results)\n except KeyboardInterrupt:\n self._progress('Interrupted')\n return self._error(_('User aborted'), info=results)\n finally:\n self._progress(\"Rescan complete\")\n del config._running['rescan']\n\n def _rescan_vcards(self, session, which):\n from mailpile.plugins import PluginManager\n config = session.config\n imported = 0\n importer_cfgs = config.prefs.vcard.importers\n which_spec = which.split(':')\n importers = []\n try:\n self._progress(_('Rescanning: %s') % 'vcards')\n for importer in PluginManager.VCARD_IMPORTERS.values():\n if (len(which_spec) > 1 and\n which_spec[1] != importer.SHORT_NAME):\n continue\n importers.append(importer.SHORT_NAME)\n for cfg in importer_cfgs.get(importer.SHORT_NAME, []):\n if cfg:\n imp = importer(session, cfg)\n self._progress(_('Importing VCards from: %s') % imp)\n imported += imp.import_vcards(session, config.vcards)\n if mailpile.util.QUITTING:\n return {\n 'vcards': imported,\n 'vcard_sources': importers,\n 'aborted': True}\n except KeyboardInterrupt:\n return {\n 'vcards': imported,\n 'vcard_sources': importers,\n 'aborted': True}\n return {\n 'vcards': imported,\n 'vcard_sources': importers}\n\n def _run_rescan_command(self, session, timeout=120):\n pre_command = session.config.prefs.rescan_command\n if pre_command and not mailpile.util.QUITTING:\n self._progress(_('Running: %s') % pre_command)\n if not ('|' in pre_command or\n '&' in pre_command or\n ';' in pre_command):\n pre_command = pre_command.split()\n cmd = subprocess.Popen(pre_command,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n shell=not isinstance(pre_command, list))\n countdown = [timeout]\n def eat(fmt, fd):\n for line in fd:\n session.ui.notify(fmt % line.strip())\n countdown[0] = timeout\n for t in [\n threading.Thread(target=eat, args=['E: %s', cmd.stderr]),\n threading.Thread(target=eat, args=['O: %s', cmd.stdout])\n ]:\n t.daemon = True\n t.start()\n try:\n while countdown[0] > 0:\n countdown[0] -= 1\n if cmd.poll() is not None:\n rv = cmd.wait()\n if rv != 0:\n session.ui.notify(_('Rescan command returned %d')\n % rv)\n return\n elif mailpile.util.QUITTING:\n return\n time.sleep(1)\n finally:\n if cmd.poll() is None:\n session.ui.notify(_('Aborting rescan command'))\n cmd.terminate()\n time.sleep(0.2)\n if cmd.poll() is None:\n cmd.kill()\n# NOTE: For some reason we were using the un-safe Popen before, not sure\n# if that matters. Leaving this commented out for now for reference.\n#\n# try:\n# MakePopenUnsafe()\n# subprocess.check_call(pre_command, shell=True)\n# finally:\n# MakePopenSafe()\n\n def _rescan_mailboxes(self, session, which='mailboxes', force=True, deadline=None):\n import mailpile.mail_source\n config = session.config\n idx = self._idx()\n msg_count = 0\n mbox_count = 0\n rv = True\n try:\n self._progress(_('Rescanning: %s') % which)\n\n self._run_rescan_command(session)\n if which.startswith('mailbox:'):\n only = which.split(':')[1]\n which = 'mailboxes'\n else:\n only = None\n\n msg_count = 1\n if which in ('both', 'mailboxes', 'editable'):\n if only or which == 'editable':\n mailboxes = config.get_mailboxes()\n else:\n # This combination of arguments will ignore mailboxes linked to\n # active mail sources, but include the local caches of sources\n # that have been disabled.\n mailboxes = config.get_mailboxes(with_mail_source=False,\n mail_source_locals=True)\n\n for fid, fpath, sc in mailboxes:\n if mailpile.util.QUITTING:\n break\n if fpath == '/dev/null':\n continue\n if only and (only != fpath) and (only != fid):\n continue\n try:\n self._progress(_('Rescanning: %s %s') % (fid, fpath))\n rescan_args = {\n 'event': self.event,\n 'deadline': deadline,\n 'force': force}\n if which == 'editable':\n count = idx.scan_mailbox(session, fid, fpath,\n config.open_mailbox,\n process_new=False,\n editable=True,\n **rescan_args)\n else:\n count = idx.scan_mailbox(session, fid, fpath,\n config.open_mailbox,\n **rescan_args)\n except ValueError:\n self._ignore_exception()\n count = -1\n if count < 0:\n session.ui.warning(_('Failed to rescan: %s') %\n FilePath(fpath).display())\n elif count > 0:\n msg_count += count\n mbox_count += 1\n session.ui.mark('\\n')\n\n if which in ('both', 'sources'):\n ocount = msg_count - 1\n while ocount != msg_count:\n ocount = msg_count\n src_ids = config.sources.keys()\n src_ids.sort(key=lambda k: random.randint(0, 100))\n for src_id in src_ids:\n try:\n src = config.get_mail_source(src_id, start=True)\n if mailpile.util.QUITTING:\n ocount = msg_count\n break\n self._progress(_('Rescanning: %s') % (src, ))\n (messages, mailboxes) = src.rescan_now(session)\n except ValueError:\n messages = mailboxes = 0\n if messages > 0:\n msg_count += messages\n mbox_count += mailboxes\n session.ui.mark('\\n')\n if not session.ui.interactive:\n break\n except (KeyboardInterrupt, subprocess.CalledProcessError) as e:\n return {\n 'aborted': True,\n 'messages': msg_count,\n 'mailboxes': mbox_count}\n finally:\n if msg_count:\n session.ui.mark('\\n')\n if msg_count < 500:\n self._background_save(index=True)\n else:\n self._background_save(index_full=True)\n else:\n self._progress(_('Nothing changed'))\n return {\n 'messages': msg_count,\n 'mailboxes': mbox_count}\n\n\nclass Optimize(Command):\n \"\"\"Optimize the keyword search index\"\"\"\n SYNOPSIS = (None, 'optimize', None, '[harder]')\n ORDER = ('Internals', 3)\n\n def command(self, slowly=False):\n try:\n if not slowly:\n mailpile.util.LAST_USER_ACTIVITY = 0\n self._idx().save(self.session)\n GlobalPostingList.Optimize(self.session, self._idx(),\n force=('harder' in self.args))\n return self._success(_('Optimized search engine'))\n except KeyboardInterrupt:\n return self._error(_('Aborted'))\n\n\nclass DeleteMessages(Command):\n \"\"\"Delete one or more messages.\"\"\"\n SYNOPSIS = (None, 'delete', 'message/delete', '[--keep] ')\n ORDER = ('Searching', 99)\n IS_USER_ACTIVITY = True\n\n def command(self, slowly=False):\n idx = self._idx()\n\n args = list(self.args)\n keep = 0\n while '--keep' in args:\n args.remove('--keep')\n keep += 1\n\n # We group messages by mailbox and delete in batches. This should\n # avoid loading all the mailboxes into RAM at once, which is a big\n # deal on larger setups.\n targets = [Email(idx, mi) for mi in self._choose_messages(args)]\n msg_ptr_pairs = [\n (e, e.index.unique_mbox_ids(e.get_msg_info())) for e in targets]\n\n if 'deletion' in self.session.config.sys.debug:\n self.session.ui.debug('Targets: %s' % msg_ptr_pairs)\n\n # Message are sorted so the ones present in the most mailboxes\n # are listed first.\n msg_ptr_pairs.sort(key=lambda mpp: (-len(mpp[1]), mpp[0]))\n\n deleted, failed = [], []\n while msg_ptr_pairs:\n # Pick the largest set of mailboxes we have yet to delete from\n mid_set = msg_ptr_pairs[0][1]\n\n # Pick all the messages contained in this set of mailboxes\n messages = [e for e, mids in msg_ptr_pairs\n if ((mid_set | mids) == mid_set)]\n\n # Go delete them!\n mailboxes = []\n for e in messages:\n msg_idx = e.msg_idx_pos\n del_ok, mboxes = e.delete_message(self.session,\n flush=False, keep=keep)\n mailboxes.extend(mboxes)\n if del_ok:\n deleted.append(msg_idx)\n else:\n failed.append(msg_idx)\n\n # This will actually delete from mboxes, etc.\n for m in set(mailboxes):\n with m:\n m.flush()\n\n # OK, these are done, reduce our target list\n msg_ptr_pairs = [(e, mids) for e, mids in msg_ptr_pairs\n if ((mid_set | mids) != mid_set)]\n\n # FIXME: Trigger a background rescan of affected mailboxes, as\n # the flush() above may have broken our pointers.\n\n result = {'deleted': deleted}\n if failed:\n result['failed'] = failed\n return self._error(_('Could not delete all messages'),\n result=result)\n return self._success(_('Deleted %d messages') % len(deleted),\n result=result)\n\n\nclass BrowseOrLaunch(Command):\n \"\"\"Launch browser and exit, if already running\"\"\"\n SYNOPSIS = (None, 'browse_or_launch', None, None)\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n RAISES = (KeyboardInterrupt,)\n\n @classmethod\n def Browse(cls, sspec):\n http_url = ('http://%s:%s%s/' % sspec\n ).replace('//0.0.0.0:', '//localhost:')\n try:\n MakePopenUnsafe()\n webbrowser.open(http_url)\n return http_url\n except:\n pass\n finally:\n MakePopenSafe()\n return False\n\n def command(self):\n config = self.session.config\n\n if config.http_worker:\n sspec = config.http_worker.sspec\n else:\n sspec = (config.sys.http_host, config.sys.http_port,\n config.sys.http_path or '')\n\n try:\n socket.create_connection(sspec[:2])\n self.Browse(sspec)\n os._exit(127)\n except IOError:\n pass\n\n return self._success(_('Launching Mailpile'), result=True)\n\n\nclass RunWWW(Command):\n \"\"\"Just run the web server\"\"\"\n SYNOPSIS = (None, 'www', None, '[]')\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n\n def command(self):\n config = self.session.config\n ospec = (config.sys.http_host, config.sys.http_port,\n config.sys.http_path)\n\n if self.args:\n host, portpath = self.args[0].split('://')[-1].split(':', 1)\n port, path = (portpath+'/').split('/', 1)\n port = int(port)\n sspec = (host, port, WebRootCheck(path))\n else:\n sspec = ospec\n\n if self.session.config.http_worker:\n self.session.config.http_worker.quit(join=True)\n self.session.config.http_worker = None\n\n self.session.config.prepare_workers(self.session,\n httpd_spec=tuple(sspec),\n daemons=True)\n if config.http_worker:\n sspec = config.http_worker.httpd.sspec\n http_url = 'http://%s:%s%s/' % sspec\n if sspec != ospec:\n (config.sys.http_host, config.sys.http_port,\n config.sys.http_path) = sspec\n self._background_save(config=True)\n return self._success(_('Moved the web server to %s'\n ) % http_url)\n else:\n return self._success(_('Started the web server on %s'\n ) % http_url)\n else:\n return self._error(_('Failed to start the web server'))\n\n\nclass Cleanup(Command):\n \"\"\"Perform cleanup actions (runs before shutdown)\"\"\"\n SYNOPSIS = (None, 'cleanup', None, \"\")\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n SPLIT_ARG = False\n TASKS = []\n\n @classmethod\n def AddTask(cls, task, last=False, first=False):\n safe_assert(not (first and last))\n if (first or last) and not cls.TASKS:\n cls.TASKS = [lambda: True]\n if first:\n cls.TASKS.insert(0, task)\n elif last:\n cls.TASKS.append(task)\n else:\n cls.TASKS.insert(len(cls.TASKS) - 1, task)\n\n def command(self):\n while self.TASKS:\n try:\n self.TASKS.pop(0)()\n except:\n traceback.print_exc()\n pass\n return self._success(_('Performed shutdown tasks'))\n\n\nclass WritePID(Command):\n \"\"\"Write the PID to a file\"\"\"\n SYNOPSIS = (None, 'pidfile', None, \"\")\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n SPLIT_ARG = False\n\n def command(self):\n filename = self.args[0]\n with vfs.open(filename, 'w') as fd:\n fd.write('%d' % os.getpid())\n Cleanup.AddTask(lambda: os.unlink(filename), last=True)\n return self._success(_('Wrote PID to %s') % self.args)\n\n\nclass RenderPage(Command):\n \"\"\"Does nothing, for use by semi-static jinja2 pages\"\"\"\n SYNOPSIS = (None, None, 'page', None)\n ORDER = ('Internals', 6)\n CONFIG_REQUIRED = False\n SPLIT_ARG = False\n HTTP_STRICT_VARS = False\n IS_USER_ACTIVITY = True\n\n def template_path(self, ttype, template_id=None, **kwargs):\n if not template_id:\n template_id = '%s/%s' % (self.SYNOPSIS[2],\n self.args and self.args[0] or '')\n return Command.template_path(self, ttype, template_id=template_id,\n **kwargs)\n\n def command(self):\n return self._success(_('Rendered the page'), result={\n 'path': (self.args and self.args[0] or ''),\n 'data': self.data\n })\n\n\nclass ProgramStatus(Command):\n \"\"\"Display list of running threads, locks and outstanding events.\"\"\"\n SYNOPSIS = (None, 'ps', 'ps', None)\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n LOG_NOTHING = True\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n now = time.time()\n\n sessions = self.result.get('sessions')\n if sessions:\n sessions = '\\n'.join(sorted([' %s/%s = %s (%ds)'\n % (us['sessionid'],\n us['userdata'],\n us['userinfo'],\n now - us['timestamp'])\n for us in sessions]))\n else:\n sessions = ' ' + _('Nothing Found')\n\n ievents = self.result.get('ievents')\n cevents = self.result.get('cevents')\n if cevents:\n cevents = '\\n'.join([' %s' % (e.as_text(compact=True),)\n for e in cevents])\n else:\n cevents = ' ' + _('Nothing Found')\n\n ievents = self.result.get('ievents')\n if ievents:\n ievents = '\\n'.join([' %s' % (e.as_text(compact=True),)\n for e in ievents])\n else:\n ievents = ' ' + _('Nothing Found')\n\n threads = self.result.get('threads')\n if threads:\n threads = '\\n'.join(sorted([(' ' + str(t)) for t in threads]))\n else:\n threads = _('Nothing Found')\n\n locks = self.result.get('locks')\n if locks:\n locks = '\\n'.join(sorted([(' %s.%s is %slocked'\n ) % (l[0], l[1],\n '' if l[2] else 'un')\n for l in locks]))\n else:\n locks = _('Nothing Found')\n\n return ('Recent events:\\n%s\\n\\n'\n 'Events in progress:\\n%s\\n\\n'\n 'Live sessions:\\n%s\\n\\n'\n 'Postinglist timers:\\n%s\\n\\n'\n 'Threads: (bg delay %.3fs, live=%s, httpd=%s)\\n%s\\n\\n'\n 'Locks:\\n%s'\n ) % (cevents, ievents, sessions,\n self.result['pl_timers'],\n self.result['delay'],\n self.result['live'],\n self.result['httpd'],\n threads, locks)\n\n def command(self, args=None):\n import mailpile.auth\n import mailpile.mail_source\n import mailpile.plugins.compose\n import mailpile.plugins.contacts\n\n config = self.session.config\n\n try:\n idx = config.index\n locks = [\n ('config.index', '_lock', idx._lock._is_owned()),\n ('config.index', '_save_lock', idx._save_lock._is_owned())\n ]\n except AttributeError:\n locks = []\n if config.vcards:\n locks.extend([\n ('config.vcards', '_lock', config.vcards._lock._is_owned()),\n ])\n locks.extend([\n ('config', '_lock', config._lock._is_owned()),\n ('mailpile.plugins.compose',\n 'GLOBAL_EDITING_LOCK',\n mailpile.plugins.compose.GLOBAL_EDITING_LOCK._is_owned()),\n ('mailpile.plugins.contacts',\n 'GLOBAL_VCARD_LOCK',\n mailpile.plugins.contacts.GLOBAL_VCARD_LOCK._is_owned()),\n ('mailpile.postinglist',\n 'PLC_CACHE_LOCK',\n mailpile.postinglist.PLC_CACHE_LOCK.locked()),\n ('mailpile.postinglist',\n 'GLOBAL_POSTING_LOCK',\n mailpile.postinglist.GLOBAL_POSTING_LOCK._is_owned()),\n ('mailpile.postinglist',\n 'GLOBAL_OPTIMIZE_LOCK',\n mailpile.postinglist.GLOBAL_OPTIMIZE_LOCK.locked()),\n ('mailpile.postinglist',\n 'GLOBAL_GPL_LOCK',\n mailpile.postinglist.GLOBAL_GPL_LOCK._is_owned())])\n\n threads = threading.enumerate()\n for thread in threads:\n try:\n if hasattr(thread, 'lock'):\n locks.append([thread, 'lock', thread.lock])\n if hasattr(thread, '_lock'):\n locks.append([thread, '_lock', thread._lock])\n if locks and hasattr(locks[-1][-1], 'locked'):\n locks[-1][-1] = locks[-1][-1].locked()\n elif locks and hasattr(locks[-1][-1], '_is_owned'):\n locks[-1][-1] = locks[-1][-1]._is_owned()\n except AttributeError:\n pass\n\n import mailpile.auth\n import mailpile.httpd\n result = {\n 'sessions': [{'sessionid': k,\n 'timestamp': v.ts,\n 'userdata': v.data,\n 'userinfo': v.auth} for k, v in\n mailpile.auth.SESSION_CACHE.iteritems()],\n 'pl_timers': mailpile.postinglist.TIMERS,\n 'delay': play_nice_with_threads(sleep=False),\n 'live': mailpile.util.LIVE_USER_ACTIVITIES,\n 'httpd': mailpile.httpd.LIVE_HTTP_REQUESTS,\n 'threads': threads,\n 'locks': sorted(locks)\n }\n if config.event_log:\n result.update({\n 'cevents': list(config.event_log.events(flag='c'))[-10:],\n 'ievents': config.event_log.incomplete(),\n })\n\n return self._success(_(\"Listed events, threads, and locks\"),\n result=result)\n\n\nclass CronStatus(Command):\n \"\"\"Manually edit or display the background job schedule\"\"\"\n SYNOPSIS = (None, 'cron', None,\n \"[ <--trigger|--interval |--postpone >]\")\n ORDER = ('Internals', 4)\n IS_USER_ACTIVITY = False\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n def _t(dt):\n return '%4.4d-%2.2d-%2.2d %2.2d:%2.2d' % (\n dt.year, dt.month, dt.day, dt.hour, dt.minute)\n\n fmt = ' %-23s %8s %-16s %-16s %s'\n lines = [\n 'Background CRON last ran at %s.' % _t(\n datetime.datetime.fromtimestamp(self.result['last_run'])),\n 'Current schedule:',\n '',\n fmt % ('JOB', 'INTERVAL', 'LAST RUN', 'NEXT RUN', 'STATUS')]\n\n for job_name, interval, func, last, status in self.result['jobs']:\n lines.append(fmt % (\n job_name,\n interval,\n (_t(datetime.datetime.fromtimestamp(last))\n if (status != 'new') else ''),\n _t(datetime.datetime.fromtimestamp(last + interval)),\n status))\n\n return '\\n'.join(lines)\n\n def command(self, args=None):\n config = self.session.config\n args = args if (args is not None) else list(self.args)\n now = int(time.time())\n\n if args:\n job = args.pop(0)\n while args:\n op = args.pop(0).lower().replace('-', '')\n if op == 'interval':\n interval = int(args.pop(0))\n config.cron_worker.schedule[job][1] = interval\n elif op == 'trigger':\n interval = config.cron_worker.schedule[job][1]\n config.cron_worker.schedule[job][3] = now - interval\n elif op == 'postpone':\n hours = float(args.pop(0))\n config.cron_worker.schedule[job][3] += int(hours * 3600)\n else:\n raise NotImplementedError('Unknown op: %s' % op)\n\n return self._success(\n _(\"Displayed CRON schedule\"),\n result={\n 'last_run': config.cron_worker.last_run,\n 'jobs': config.cron_worker.schedule.values()})\n\n\nclass HealthCheck(Command):\n \"\"\"Check and report app health\"\"\"\n SYNOPSIS = (None, 'health', None, \"\")\n ORDER = ('Internals', 4)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n\n # We cache our health event, so it can be updated by the class methods.\n health_event = None\n\n def _create_event(self):\n if HealthCheck.health_event is not None:\n self.event = HealthCheck.health_event\n else:\n Command._create_event(self)\n self.event.data['starttime'] = int(time.time())\n self.event.data['problems'] = {}\n self.event.data['healthy'] = True\n HealthCheck.health_event = self.event\n\n # Cancel any obsolete HealthCheck events we find\n if self.session.config.event_log:\n for ev in self.session.config.event_log.events():\n if (ev.source == self.event.source and\n ev.event_id != self.event.event_id):\n ev.flags = ev.COMPLETE\n self.session.config.event_log.log_event(ev)\n\n @classmethod\n def _mem_check(cls, session, config):\n if config.detected_memory_corruption:\n return _('Memory corruption detected') + '!'\n return False\n\n @classmethod\n def _disk_check(cls, session, config):\n if config.need_more_disk_space():\n return _('Insufficient free disk space') + '.'\n return False\n\n @classmethod\n def _readonly_check(cls, session, config):\n from mailpile.security import _lockdown_basic\n if _lockdown_basic(config):\n return _('Your Mailpile is read-only!')\n return False\n\n @classmethod\n def check(cls, session, config):\n # Check all the things! The order here matters, more critical things\n # should be reported last as they will determine the final message.\n if not cls.health_event:\n return False\n\n messages = []\n problems = cls.health_event.data['problems']\n\n was_healthy = cls.health_event.data['healthy']\n old_problems = ' '.join(sorted(problems.keys()))\n\n now_healthy = True\n for crit, name, check in ((True, 'disk', cls._disk_check),\n (True, 'memcheck', cls._mem_check),\n (True, 'readonly', cls._readonly_check)):\n message = check(session, config)\n if message:\n problems[name] = message\n messages.append(message)\n if crit:\n now_healthy = False\n elif name in problems:\n del problems[name]\n\n cls.health_event.data['healthy'] = now_healthy\n if messages:\n cls.health_event.message = ' '.join(messages[-2:])\n cls.health_event.flags = cls.health_event.RUNNING\n else:\n cls.health_event.message = _('We are healthy!')\n cls.health_event.flags = cls.health_event.COMPLETE\n\n # Only record changes to the event log\n new_problems = ' '.join(sorted(problems.keys()))\n if old_problems != new_problems and config.event_log:\n config.event_log.log_event(cls.health_event)\n\n return True\n\n def command(self, args=None):\n self.check(self.session, self.session.config)\n return self._success(self.event.message, result=self.event)\n\n\nclass GpgCommand(Command):\n \"\"\"Interact with GPG directly\"\"\"\n SYNOPSIS = (None, 'gpg', None, \"\")\n ORDER = ('Internals', 4)\n IS_USER_ACTIVITY = True\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n if self.result:\n return '%s\\n\\n%s' % (\n (self.result['stdout'] or _('(no output)')).strip(),\n self.message)\n return '%s' % self.message\n\n def command(self, args=None):\n args = list((args is None) and self.args or args or [])\n gnupg = self._gnupg()\n gnupg_args = gnupg.common_args(interactive=True)\n binary = gnupg.gpgbinary\n rv = '(unknown)'\n def _shellquote(s):\n return \"'\" + s.replace(\"'\", \"'\\\\''\") + \"'\"\n with self.session.ui.term:\n try:\n self.session.ui.block()\n if (self.session.ui.interactive and\n self.session.ui.render_mode == 'text'):\n rv = os.system(' '.join(_shellquote(a)\n for a in (gnupg_args + args)))\n stdout = None\n else:\n sp = subprocess.Popen(\n gnupg_args + ['--batch', '--no-tty'] + args,\n stdout=subprocess.PIPE, stderr=subprocess.PIPE,\n stdin=subprocess.PIPE)\n (stdout, stderr) = sp.communicate(input='')\n rv = sp.wait()\n from mailpile.plugins.vcard_gnupg import PGPKeysImportAsVCards\n PGPKeysImportAsVCards(self.session).run()\n except:\n self.session.ui.unblock()\n\n return self._success(_(\"That was fun!\") + ' ' +\n _(\"%s returned: %s\") % (binary, rv),\n result={'binary': binary,\n 'stdout': stdout,\n 'returned': rv})\n\n\nclass ListDir(Command):\n \"\"\"Display working directory listing\"\"\"\n SYNOPSIS = (None, 'ls', 'browse', \"[-a] [-d] [ ...]\")\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = True\n COMMAND_SECURITY = security.CC_BROWSE_FILESYSTEM\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n if self.result and self.result['entries']:\n lines = []\n for i in self.result['entries']:\n sz = i.get('bytes')\n dn = i['display_name']\n dp = '' if (i['display_path'].endswith(dn)\n ) else i['display_path']\n dn += '/' if i.get('flag_directory') else ''\n lines.append(('%12.12s %s%-20s %s'\n ) % ('' if (sz is None) else sz,\n '>' if i.get('flag_mailsource') else\n '*' if i.get('flag_mailbox') else ' ',\n dn, dp))\n return '\\n'.join(lines)\n else:\n return _('Nothing Found')\n\n def command(self, args=None):\n args = list((args is None) and self.args or args or [])\n flags = [f for f in args if f[:1] == '-']\n args = [a for a in args if a[:1] != '-']\n\n if '_method' in self.data:\n args = ['/' + '/'.join(args)]\n\n if not args:\n args = ['.']\n\n def lsf(f):\n info = {'path': f}\n try:\n info = vfs.getinfo(f, self.session.config)\n info['icon'] = ''\n for k in info.get('flags', []):\n info['flag_%s' % unicode(k).lower().replace('.', '_')\n ] = True\n except (OSError, IOError, UnicodeDecodeError):\n info['flag_error'] = True\n return info\n def ls(p):\n return [lsf(vfs.path_join(p, f)) for f in vfs.listdir(p)\n if '-a' in flags or f.raw_fp[:1] != '.']\n\n file_list = []\n errors = 0\n for path in args:\n if (security.forbid_command(self, security.CC_ACCESS_FILESYSTEM)\n and (path != '/')\n and (not path.endswith('$'))\n and ('$/' not in path)):\n continue\n try:\n path = os.path.expanduser(path.encode('utf-8'))\n if vfs.isdir(path) and '*' not in path:\n file_list.extend(ls(path))\n else:\n for p in vfs.glob(path):\n if vfs.isdir(p) and '-d' not in flags:\n file_list.extend(ls(p))\n else:\n file_list.append(lsf(p))\n except (socket.error, socket.gaierror) as e:\n return self._error(_('Network error: %s') % e)\n except (OSError, IOError, UnicodeDecodeError) as e:\n errors += 1\n\n if errors and not file_list:\n traceback.print_exc()\n return self._error(_('Failed to list: %s') % e)\n\n id_src_map = self.session.config.find_mboxids_and_sources_by_path(\n *[unicode(f['path']) for f in file_list])\n for info in file_list:\n path = unicode(info['path'])\n mid_src = id_src_map.get(path)\n if mid_src:\n mid, src = mid_src\n if src:\n info['source'] = src._key\n if src and src.mailbox[mid] and src.mailbox[mid].primary_tag:\n tid = src.mailbox[mid].primary_tag\n if tid in self.session.config.tags:\n info['tag'] = self.session.config.tags[tid].slug\n info['icon'] = self.session.config.tags[tid].icon\n elif info.get('flag_mailsource'):\n if path.startswith('/src:'):\n info['source'] = path[5:]\n\n file_list.sort(key=lambda i: i['display_path'].lower())\n return self._success(_('Listed %d files or directories'\n ) % len(file_list),\n result={\n 'path': args[0] if (len(args) == 1) else args,\n 'name': vfs.display_name(args[0], self.session.config),\n 'entries': file_list\n })\n\n\nclass ChangeDir(ListDir):\n \"\"\"Change working directory\"\"\"\n SYNOPSIS = (None, 'cd', None, \"<.../new/path/...>\")\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = True\n COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM\n\n def command(self, args=None):\n try:\n args = list((args is None) and self.args or args or [])\n os.chdir(FilePath.unalias(\n os.path.expanduser(args.pop(0).encode('utf-8'))))\n return ListDir.command(self, args=['.'])\n except (OSError, IOError, UnicodeEncodeError) as e:\n return self._error(_('Failed to change directories: %s') % e)\n\n\nclass CatFile(Command):\n \"\"\"Dump the contents of a file, decrypting if necessary\"\"\"\n SYNOPSIS = (None, 'cat', None, \" [>/path/to/output]\")\n ORDER = ('Internals', 5)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = True\n COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM\n\n class CommandResult(Command.CommandResult):\n def as_text(self):\n if isinstance(self.result, list):\n return ''.join(self.result)\n else:\n return ''\n\n def command(self, args=None):\n lines = []\n files = list(args or self.args)\n target = tfd = None\n if files and files[-1] and files[-1][:1] == '>':\n target = files.pop(-1)[1:]\n if vfs.exists(target):\n return self._error(_('That file already exists: %s'\n ) % target)\n tfd = vfs.open(target, 'wb')\n cb = lambda ll: [tfd.write(l) for l in ll]\n else:\n cb = lambda ll: lines.extend((l.decode('utf-8') for l in ll))\n\n for fn in files:\n with vfs.open(fn, 'r') as fd:\n def errors(where):\n self.session.ui.error('Decrypt failed at %d' % where)\n decrypt_and_parse_lines(fd, cb, self.session.config,\n newlines=True, decode=None,\n gpgi=self._gnupg(),\n _raise=False, error_cb=errors)\n\n if tfd:\n tfd.close()\n return self._success(_('Dumped to %s: %s'\n ) % (target, ', '.join(files)))\n else:\n return self._success(_('Dumped: %s') % ', '.join(files),\n result=lines)\n\n\n##[ Configuration commands ]###################################################\n\n\nclass ListLanguages(Command):\n \"\"\"List available languages\"\"\"\n SYNOPSIS = (None, 'languages', 'settings/languages', '')\n ORDER = ('Config', 1)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n HTTP_CALLABLE = ('GET', )\n\n def command(self):\n from mailpile.i18n import ListTranslations\n langs = ListTranslations(self.session.config)\n return self._success(_('Listed available translations'),\n result=sorted([(l, langs[l]) for l in langs]))\n\n\nclass ConfigSet(Command):\n \"\"\"Change a setting\"\"\"\n SYNOPSIS = ('S', 'set', 'settings/set',\n '[--force] ')\n ORDER = ('Config', 1)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n\n SPLIT_ARG = False\n\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_STRICT_VARS = False\n HTTP_POST_VARS = {\n '_section': 'common section, create if needed',\n 'section.variable': 'value|json-string'\n }\n\n def command(self):\n from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD\n\n config = self.session.config\n args = list(self.args)\n arg = ' '.join(args)\n ops = []\n on_cli = (self.data.get('_method', 'CLI') == 'CLI')\n force = False\n\n if arg.startswith('--force '):\n if not on_cli:\n raise ValueError('The --force flag only works on the CLI')\n force = True\n arg = arg[8:]\n\n if not force:\n fb = security.forbid_command(self, security.CC_CHANGE_CONFIG)\n if fb:\n return self._error(fb)\n\n if not config.loaded_config:\n self.session.ui.warning(_('WARNING: Any changes will '\n 'be overwritten on login'))\n\n section = self.data.get('_section', [''])[0]\n if section:\n # Make sure section exists\n ops.append((section, '!CREATE_SECTION'))\n\n for var in self.data.keys():\n if (var in ('_section', '_method', 'context', 'csrf')\n or var.startswith('ui_')):\n continue\n sep = '/' if ('/' in (section+var)) else '.'\n svar = (section+sep+var) if section else var\n parts = svar.split(sep)\n if parts[0] in config.rules:\n if svar.endswith('[]'):\n ops.append((svar[:-2], json.dumps(self.data[var])))\n else:\n ops.append((svar, self.data[var][0]))\n else:\n raise ValueError(_('Invalid section or variable: %s') % var)\n\n if args:\n if '=' in arg:\n # Backwards compatiblity with the old 'var = value' syntax.\n var, value = [s.strip() for s in arg.split('=', 1)]\n var = var.replace(': ', '.').replace(':', '.').replace(' ', '')\n else:\n var, value = arg.split(' ', 1)\n ops.append((var, value))\n\n # Access controls...\n if not force:\n for path, value in ops:\n fb = security.forbid_config_change(config, path)\n if fb:\n return self._error(fb)\n elif path == 'master_key' and config.get_master_key():\n return self._error(_('I refuse to change the master key!'))\n\n # We don't have transactions really, but making sure the HTTPD\n # is idle (aside from this request) will definitely help.\n with BLOCK_HTTPD_LOCK, Idle_HTTPD():\n updated = {}\n for path, value in ops:\n if not force:\n if path == 'master_key' and config.get_master_key():\n raise ValueError('Need --force to change master key.')\n if path == 'sys.http_no_auth':\n raise ValueError('Need --force to change auth policy.')\n\n value = value.strip()\n if value == '{None}':\n value = None\n elif value == '{Blank}':\n value = ''\n elif value == '{False}':\n value = False\n elif value == '{True}':\n value = True\n elif value[:1] in ('{', '[') and value[-1:] in ( ']', '}'):\n value = json.loads(value)\n try:\n try:\n cfg, var = config.walk(path.strip(), parent=1)\n if value == '!CREATE_SECTION':\n if var not in cfg:\n cfg[var] = {}\n else:\n cfg[var] = value\n updated[path] = value\n except IndexError:\n cfg, v1, v2 = config.walk(path.strip(), parent=2)\n cfg[v1] = {v2: value}\n except TypeError:\n raise ValueError('Could not set variable: %s' % path)\n\n if config.loaded_config:\n self._background_save(config=True)\n\n return self._success(_('Updated your settings'), result=updated)\n\n\nclass ConfigAdd(Command):\n \"\"\"Add a new value to a list (or ordered dict) setting\"\"\"\n SYNOPSIS = (None, 'append', 'settings/add', ' ')\n ORDER = ('Config', 1)\n SPLIT_ARG = False\n HTTP_CALLABLE = ('POST', 'UPDATE')\n HTTP_STRICT_VARS = False\n HTTP_POST_VARS = {\n 'section.variable': 'value|json-string',\n }\n IS_USER_ACTIVITY = True\n COMMAND_SECURITY = security.CC_CHANGE_CONFIG\n\n def command(self):\n from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD\n config = self.session.config\n args = list(self.args)\n ops = []\n\n for var in self.data.keys():\n parts = ('.' in var) and var.split('.') or var.split('/')\n if parts[0] in config.rules:\n ops.append((var, self.data[var][0]))\n\n if args:\n arg = ' '.join(args)\n if '=' in arg:\n # Backwards compatible with the old 'var = value' syntax.\n var, value = [s.strip() for s in arg.split('=', 1)]\n var = var.replace(': ', '.').replace(':', '.').replace(' ', '')\n else:\n var, value = arg.split(' ', 1)\n ops.append((var, value))\n\n # Access controls...\n for path, value in ops:\n fb = security.forbid_config_change(config, path)\n if fb:\n return self._error(fb)\n elif path == 'master_key' and config.get_master_key():\n return self._error(_('I refuse to change the master key!'))\n\n # We don't have transactions really, but making sure the HTTPD\n # is idle (aside from this request) will definitely help.\n with BLOCK_HTTPD_LOCK, Idle_HTTPD():\n updated = {}\n for path, value in ops:\n value = value.strip()\n if value.startswith('{') or value.startswith('['):\n value = json.loads(value)\n cfg, var = config.walk(path.strip(), parent=1)\n cfg[var].append(value)\n updated[path] = value\n\n if updated:\n self._background_save(config=True)\n\n return self._success(_('Updated your settings'), result=updated)\n\n\nclass ConfigUnset(Command):\n \"\"\"Reset one or more settings to their defaults\"\"\"\n SYNOPSIS = ('U', 'unset', 'settings/unset', '')\n ORDER = ('Config', 2)\n HTTP_CALLABLE = ('POST', )\n HTTP_POST_VARS = {\n 'var': 'section.variables'\n }\n IS_USER_ACTIVITY = True\n COMMAND_SECURITY = security.CC_CHANGE_CONFIG\n\n def command(self):\n from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD\n session, config = self.session, self.session.config\n\n def unset(cfg, key):\n if isinstance(cfg[key], dict):\n if '_any' in cfg[key].rules:\n for skey in cfg[key].keys():\n del cfg[key][skey]\n else:\n for skey in cfg[key].keys():\n unset(cfg[key], skey)\n elif isinstance(cfg[key], list):\n cfg[key] = []\n else:\n del cfg[key]\n\n # Access controls...\n vlist = list(self.args) + (self.data.get('var', None) or [])\n # Access controls...\n for v in vlist:\n fb = security.forbid_config_change(config, v)\n if fb:\n return self._error(fb)\n elif v == 'master_key' and config.get_master_key():\n return self._error(_('I refuse to change the master key!'))\n\n # We don't have transactions really, but making sure the HTTPD\n # is idle (aside from this request) will definitely help.\n with BLOCK_HTTPD_LOCK, Idle_HTTPD():\n updated = []\n for v in vlist:\n cfg, vn = config.walk(v, parent=True)\n unset(cfg, vn)\n updated.append(v)\n\n if updated:\n self._background_save(config=True)\n\n return self._success(_('Reset to default values'), result=updated)\n\n\nclass ConfigPrint(Command):\n \"\"\"Print one or more settings\"\"\"\n SYNOPSIS = ('P', 'print', 'settings', '[-short|-secrets|-flat] ')\n ORDER = ('Config', 3)\n CONFIG_REQUIRED = False\n IS_USER_ACTIVITY = False\n\n HTTP_CALLABLE = ('GET', 'POST')\n HTTP_QUERY_VARS = {\n 'var': 'section.variable',\n 'short': 'Set True to omit unchanged values (defaults)',\n 'secrets': 'Set True to show passwords and other secrets'\n }\n HTTP_POST_VARS = {\n 'user': 'Authenticate as user',\n 'pass': 'Authenticate with password'\n }\n\n def _maybe_all(self, list_all, data, key_types, recurse, sanitize):\n if isinstance(data, (dict, list)) and list_all:\n rv = {}\n for key in data.all_keys():\n if [t for t in data.key_types(key) if t not in key_types]:\n # Silently omit things that are considered sensitive\n continue\n rv[key] = data[key]\n if hasattr(rv[key], 'all_keys'):\n if recurse:\n rv[key] = self._maybe_all(True, rv[key], key_types,\n recurse, sanitize)\n else:\n if 'name' in rv[key]:\n rv[key] = '{ ..(%s).. }' % rv[key]['name']\n elif 'description' in rv[key]:\n rv[key] = '{ ..(%s).. }' % rv[key]['description']\n elif 'host' in rv[key]:\n rv[key] = '{ ..(%s).. }' % rv[key]['host']\n else:\n rv[key] = '{ ... }'\n elif (sanitize\n and key.lower()[:4] in ('pass', 'secr', 'obfu')):\n rv[key] = '(SUPPRESSED)'\n return rv\n return data\n\n def command(self):\n session, config = self.session, self.session.config\n result = {}\n invalid = []\n\n args = list(self.args)\n recurse = not self.data.get('flat', ['-flat' in args])[0]\n list_all = not self.data.get('short', ['-short' in args])[0]\n sanitize = not self.data.get('secrets', ['-secrets' in args])[0]\n\n if security.forbid_command(self, security.CC_LIST_PRIVATE_DATA):\n sanitize = True\n\n # FIXME: Shouldn't we suppress critical variables as well?\n key_types = ['public', 'critical']\n access_denied = False\n\n if self.data.get('_method') == 'POST':\n if 'pass' in self.data:\n from mailpile.auth import CheckPassword\n password = self.data['pass'][0]\n auth_user = CheckPassword(config,\n self.data.get('user', [None])[0],\n password)\n if auth_user == 'DEFAULT':\n key_types += ['key']\n result['_auth_user'] = auth_user\n result['_auth_pass'] = password\n\n for key in (args + self.data.get('var', [])):\n if key in ('-short', '-flat', '-secrets'):\n continue\n try:\n data = config.walk(key, key_types=key_types)\n result[key] = self._maybe_all(list_all, data, key_types,\n recurse, sanitize)\n except AccessError:\n access_denied = True\n invalid.append(key)\n except KeyError:\n invalid.append(key)\n\n if invalid:\n return self._error(_('Invalid keys'),\n result=result, info={\n 'keys': invalid,\n 'key_types': key_types,\n 'access_denied': access_denied\n })\n else:\n return self._success(_('Displayed settings'), result=result)\n\n\nclass ConfigureMailboxes(Command):\n \"\"\"\n Add one or more mailboxes.\n\n If not account is specified, the mailbox is only assigned an ID for use\n in the metadata index.\n\n If an account is specified, the mailbox will be assigned to that account\n and configured for automatic indexing.\n \"\"\"\n SYNOPSIS = ('A', 'add', 'settings/mailbox',\n '[+] [--