Repository: mailpile/Mailpile Branch: master Commit: 741e610f3e76 Files: 624 Total size: 5.2 MB Directory structure: gitextract_dkjlc6xq/ ├── .coveragerc ├── .dockerignore ├── .gitignore ├── .gitmodules ├── .travis.yml ├── .tx/ │ └── config ├── AGPLv3.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── COPYING.md ├── DEV_FAQ.md ├── Dockerfile ├── Dockerfile.dev ├── Gruntfile.js ├── MANIFEST.in ├── Makefile ├── README.md ├── babel.cfg ├── bower.json ├── docker-compose.dev.yml ├── docker-compose.yml ├── install_hooks.py ├── mailpile/ │ ├── __init__.py │ ├── __main__.py │ ├── app.py │ ├── auth.py │ ├── command_cache.py │ ├── commands.py │ ├── config/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── defaults.py │ │ ├── detect.py │ │ ├── manager.py │ │ ├── paths.py │ │ └── validators.py │ ├── conn_brokers.py │ ├── crypto/ │ │ ├── __init__.py │ │ ├── aes_utils.py │ │ ├── autocrypt.py │ │ ├── gpgi.py │ │ ├── keyinfo.py │ │ ├── mime.py │ │ ├── records.py │ │ ├── state.py │ │ ├── streamer.py │ │ └── tor.py │ ├── eventlog.py │ ├── httpd.py │ ├── i18n.py │ ├── index/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── mailboxes.py │ │ ├── msginfo.py │ │ └── search.py │ ├── mail_source/ │ │ ├── __init__.py │ │ ├── imap.py │ │ ├── imap_starttls.py │ │ ├── imap_utf7.py │ │ ├── local.py │ │ └── pop3.py │ ├── mailboxes/ │ │ ├── __init__.py │ │ ├── gmvault.py │ │ ├── macmail.py │ │ ├── maildir.py │ │ ├── maildirwin.py │ │ ├── mbox.py │ │ ├── pop3.py │ │ └── wervd.py │ ├── mailutils/ │ │ ├── __init__.py │ │ ├── addresses.py │ │ ├── emails.py │ │ ├── generator.py │ │ ├── header.py │ │ ├── headerprint.py │ │ ├── html.py │ │ ├── safe.py │ │ └── vcal.py │ ├── packing.py │ ├── platforms.py │ ├── plugins/ │ │ ├── __init__.py │ │ ├── autotag.py │ │ ├── autotag_sb.py │ │ ├── backups.py │ │ ├── compose.py │ │ ├── contacts.py │ │ ├── core.py │ │ ├── crypto_autocrypt.py │ │ ├── crypto_gnupg.py │ │ ├── crypto_policy.py │ │ ├── cryptostate.py │ │ ├── dates.py │ │ ├── eventlog.py │ │ ├── exporters.py │ │ ├── groups.py │ │ ├── gui.py │ │ ├── html_magic.py │ │ ├── keylookup/ │ │ │ ├── __init__.py │ │ │ ├── email_keylookup.py │ │ │ └── wkd.py │ │ ├── migrate.py │ │ ├── motd.py │ │ ├── oauth.py │ │ ├── plugins.py │ │ ├── search.py │ │ ├── setup_magic.py │ │ ├── setup_magic_ispdb.py │ │ ├── sizes.py │ │ ├── smtp_server.py │ │ ├── tags.py │ │ ├── vcard_carddav.py │ │ ├── vcard_gnupg.py │ │ ├── vcard_gravatar.py │ │ ├── vcard_libravatar.py │ │ ├── vcard_mork.py │ │ └── webterminal.py │ ├── postinglist.py │ ├── safe_popen.py │ ├── search.py │ ├── search_history.py │ ├── security.py │ ├── smtp_client.py │ ├── spambayes/ │ │ ├── LICENSE.txt │ │ ├── Options.py │ │ ├── OptionsClass.py │ │ ├── Tester.py │ │ ├── __init__.py │ │ ├── chi2.py │ │ ├── classifier.py │ │ └── safepickle.py │ ├── tests/ │ │ ├── TESTS │ │ ├── __init__.py │ │ ├── data/ │ │ │ ├── Maildir/ │ │ │ │ ├── cur/ │ │ │ │ │ ├── .gitkeep │ │ │ │ │ ├── 1379857166.25979_1.hottie,2,S │ │ │ │ │ ├── 1379857166.25979_3.hottie,2,S │ │ │ │ │ ├── 1379857166.25979_5.hottie,2,S │ │ │ │ │ ├── 1379857166.25979_7.hottie,2,S │ │ │ │ │ ├── 1379857166.25979_9.hottie,2,S │ │ │ │ │ ├── 1379857166.25980_9.hottie,2,S │ │ │ │ │ ├── 1379857166.25981_10.hottie,2,S │ │ │ │ │ ├── broken_email_1396694612 │ │ │ │ │ ├── encrypted-empty-content-w-key.eml │ │ │ │ │ ├── fw-question-1443197278.mbx │ │ │ │ │ ├── mailpile-1398950855.mbx │ │ │ │ │ ├── mailpile-1400069992.mbx │ │ │ │ │ └── no-subject-1400067892.mbx │ │ │ │ ├── new/ │ │ │ │ │ └── .gitkeep │ │ │ │ └── tmp/ │ │ │ │ └── .gitkeep │ │ │ ├── contacts/ │ │ │ │ ├── README │ │ │ │ ├── contacttest.ldif │ │ │ │ ├── contacttest.mork │ │ │ │ └── contacttest.vcf │ │ │ ├── gpg-keyring/ │ │ │ │ ├── pubring.gpg │ │ │ │ ├── secring.gpg │ │ │ │ ├── testing.gpg.pub │ │ │ │ └── trustdb.gpg │ │ │ ├── pgp-data/ │ │ │ │ ├── buildexamples.py │ │ │ │ └── sources/ │ │ │ │ ├── ar.iso-8859-6 │ │ │ │ ├── ar.utf-8 │ │ │ │ ├── ar.windows-1256 │ │ │ │ ├── cn.utf-8 │ │ │ │ ├── gr.iso-8859-7 │ │ │ │ ├── gr.utf-8 │ │ │ │ ├── gr.windows-1253 │ │ │ │ ├── ru.koi8-ru │ │ │ │ ├── ru.utf-8 │ │ │ │ ├── ru.windows-1251 │ │ │ │ └── vi.utf-8 │ │ │ ├── pub.key │ │ │ └── tests.mbx │ │ ├── gui/ │ │ │ ├── __init__.py │ │ │ ├── test_contacts.py │ │ │ ├── test_mail.py │ │ │ └── test_tags.py │ │ ├── test_command.py │ │ ├── test_config.py │ │ ├── test_crypto_policy.py │ │ ├── test_eventlog.py │ │ ├── test_keylookup.py │ │ ├── test_mail_generator.py │ │ ├── test_mailutils.py │ │ ├── test_performance.py │ │ ├── test_plugin.py │ │ ├── test_search.py │ │ ├── test_ui.py │ │ ├── test_vcard.py │ │ └── test_vcard_mork.py │ ├── ui.py │ ├── urlmap.py │ ├── util.py │ ├── vcard.py │ ├── vfs.py │ ├── workers.py │ └── www/ │ ├── __init__.py │ ├── jinjaextensions.py │ └── jinjaloader.py ├── mp.cmd ├── package.json ├── packages/ │ ├── Dockerfile_debian │ ├── debian/ │ │ ├── changelog │ │ ├── compat │ │ ├── control │ │ ├── copyright │ │ ├── gbp.conf │ │ ├── mailpile-apache2.dirs │ │ ├── mailpile-apache2.install │ │ ├── mailpile-apache2.links │ │ ├── mailpile-apache2.lintian-overrides │ │ ├── mailpile-apache2.postinst │ │ ├── mailpile-apache2.postrm │ │ ├── mailpile-desktop.install │ │ ├── mailpile-desktop.links │ │ ├── mailpile.install │ │ ├── mailpile.lintian-overrides │ │ ├── mailpile.manpages │ │ ├── rules │ │ ├── source/ │ │ │ └── format │ │ └── watch │ ├── docker/ │ │ └── entrypoint.sh │ ├── macos/ │ │ ├── README.md │ │ ├── appdmg.json.template │ │ ├── brew/ │ │ │ └── symlinks.rb │ │ ├── build-script/ │ │ │ ├── 00-check-dependencies │ │ │ ├── 10-create-app-icons │ │ │ ├── 11-build-gui-o-mac-tic │ │ │ ├── 20-build-homebrew │ │ │ ├── 21-install-mailpile-deps │ │ │ ├── 30-slim-down │ │ │ ├── 55-install-mailpile │ │ │ ├── 90-fix-libraries │ │ │ ├── 91-fix-paths │ │ │ ├── 92-fix-python-launcher │ │ │ ├── 98-codesign │ │ │ └── 99-make-dmg │ │ ├── build.sh │ │ ├── configurator.sh │ │ ├── dmgbuild-settings.py │ │ └── mailpile │ ├── mailpile.1 │ ├── redhat/ │ │ └── mailpile.spec.in │ ├── scripts/ │ │ ├── build-controller.py │ │ ├── build-deb.sh │ │ ├── build-desktop.py │ │ ├── crontab │ │ ├── deb-package-py-deps.sh │ │ ├── kite-runner-mac.cfg │ │ ├── kite-runner-sample.cfg │ │ ├── kite-runner-win.cfg │ │ ├── kite-runner.py │ │ ├── kiterunner.service.plist │ │ └── update-repos.sh │ └── windows-wix/ │ ├── README.QUICK_START.md │ ├── README.md │ ├── TODO.md │ ├── assets/ │ │ └── LicenseText.rtf │ ├── bin/ │ │ ├── launch-mailpile.bat │ │ └── with-mailpile-env.py │ ├── dependencies-windows.txt │ ├── package.json │ ├── package.py │ ├── package_template.json │ ├── provide/ │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── build.py │ │ ├── cache.py │ │ ├── default.py │ │ └── scripts/ │ │ ├── __init__.py │ │ ├── bootstrap.py │ │ ├── export.py │ │ ├── external.py │ │ ├── extract_msi.py │ │ ├── git_checkout.py │ │ ├── msi_package.py │ │ ├── python27.py │ │ ├── root.py │ │ ├── sign_tree.py │ │ ├── util.py │ │ ├── version.py │ │ ├── win4gpg.py │ │ └── zip.py │ ├── provide-example-bootstrap-local.json │ ├── provide-example-bootstrap-remote.json │ ├── provide-example-fork-local.json │ ├── provide-example-fork-remote.json │ ├── provide.json │ └── resources.json ├── requirements-dev.txt ├── requirements-with-deps.txt ├── requirements.txt ├── scripts/ │ ├── __init__.py │ ├── add-gpgme-and-gnupg-to-venv │ ├── clear-cache.sh │ ├── colorprints.py │ ├── compile-messages.sh │ ├── create-debian-changelog.py │ ├── docker-dev/ │ │ ├── down │ │ ├── shell │ │ └── up │ ├── email-parsing-test.py │ ├── gitwhere.sh │ ├── gpg │ ├── less-compiler.in │ ├── mailpile │ ├── mailpile-admin │ ├── mailpile-decrypt.py │ ├── mailpile-pyside.py │ ├── mailpile-test.py │ ├── make-messages.sh │ ├── mbox-minimal.py │ ├── minimize-pgp-key.py │ ├── mk-credits.py │ ├── nginx.conf │ ├── python-lint.sh │ ├── reset-mac-mailpile.sh │ ├── setup-test.sh │ ├── test-sendmail.sh │ ├── unsent-mail-finder.py │ └── version.py ├── setup.cfg ├── setup.py ├── shared-data/ │ ├── contrib/ │ │ ├── README.md │ │ ├── autoajax/ │ │ │ ├── autoajax.js │ │ │ └── manifest.json │ │ ├── datadig/ │ │ │ ├── datadig-modal.html │ │ │ ├── datadig.html │ │ │ ├── datadig.js │ │ │ ├── datadig.py │ │ │ └── manifest.json │ │ ├── demos/ │ │ │ ├── demos.css │ │ │ ├── demos.js │ │ │ ├── demos.py │ │ │ ├── manifest.json │ │ │ ├── md5sum-wordy.html │ │ │ ├── md5sum.html │ │ │ ├── md5sum.xml │ │ │ └── search.html │ │ ├── experiments/ │ │ │ ├── experiments.py │ │ │ └── manifest.json │ │ ├── forcegrapher/ │ │ │ ├── README.md │ │ │ ├── d3.drasl.js │ │ │ ├── d3.js │ │ │ ├── forcegrapher.css │ │ │ ├── forcegrapher.html │ │ │ ├── forcegrapher.js │ │ │ ├── forcegrapher.py │ │ │ └── manifest.json │ │ ├── hacks/ │ │ │ ├── hacks.py │ │ │ └── manifest.json │ │ ├── hints/ │ │ │ ├── hints/ │ │ │ │ ├── autotagging.html │ │ │ │ ├── backups.html │ │ │ │ ├── deletion.html │ │ │ │ ├── dragging-tags.html │ │ │ │ ├── gravatar.html │ │ │ │ ├── keyboard-shortcuts.html │ │ │ │ ├── organize-sidebar.html │ │ │ │ └── spam.html │ │ │ ├── hints.html │ │ │ ├── hints.js │ │ │ ├── hints.py │ │ │ └── manifest.json │ │ ├── i18nhelper/ │ │ │ ├── i18nhelper.js │ │ │ ├── i18nhelper.py │ │ │ └── manifest.json │ │ ├── maildeck/ │ │ │ ├── maildeck.css │ │ │ ├── maildeck.html │ │ │ ├── maildeck.js │ │ │ ├── maildeck.py │ │ │ └── manifest.json │ │ ├── remoteaccess/ │ │ │ ├── manifest.json │ │ │ └── settings-remote.html │ │ └── unthread/ │ │ ├── i18n-stubs.html │ │ ├── manifest.json │ │ ├── modal.html │ │ └── unthread.js │ ├── default-theme/ │ │ ├── README.md │ │ ├── css/ │ │ │ ├── default.css │ │ │ ├── guide.css │ │ │ └── print.css │ │ ├── html/ │ │ │ ├── abortabortabort/ │ │ │ │ └── index.html │ │ │ ├── auth/ │ │ │ │ ├── login/ │ │ │ │ │ └── index.html │ │ │ │ ├── logout/ │ │ │ │ │ └── index.html │ │ │ │ └── shared.html │ │ │ ├── backup/ │ │ │ │ └── restore/ │ │ │ │ └── index.html │ │ │ ├── browse/ │ │ │ │ └── index.html │ │ │ ├── contacts/ │ │ │ │ ├── add/ │ │ │ │ │ └── index.html │ │ │ │ ├── import/ │ │ │ │ │ └── index.html │ │ │ │ ├── index.html │ │ │ │ └── view/ │ │ │ │ └── index.html │ │ │ ├── crypto/ │ │ │ │ ├── gpg/ │ │ │ │ │ └── key/ │ │ │ │ │ └── index.html │ │ │ │ └── tls/ │ │ │ │ └── getcert/ │ │ │ │ └── index.html │ │ │ ├── filter/ │ │ │ │ └── list/ │ │ │ │ └── index.html │ │ │ ├── group/ │ │ │ │ ├── add/ │ │ │ │ │ └── index.html │ │ │ │ └── index.html │ │ │ ├── help/ │ │ │ │ ├── bottom.html │ │ │ │ └── index.html │ │ │ ├── jsapi/ │ │ │ │ ├── app.js │ │ │ │ ├── compose/ │ │ │ │ │ ├── attachments.js │ │ │ │ │ ├── autosave.js │ │ │ │ │ ├── body.js │ │ │ │ │ ├── complete.js │ │ │ │ │ ├── crypto.js │ │ │ │ │ ├── events.js │ │ │ │ │ ├── init.js │ │ │ │ │ ├── modals.js │ │ │ │ │ ├── recipients.js │ │ │ │ │ └── tooltips.js │ │ │ │ ├── contacts/ │ │ │ │ │ ├── display_modes.js │ │ │ │ │ ├── events.js │ │ │ │ │ ├── init.js │ │ │ │ │ └── modals.js │ │ │ │ ├── crypto/ │ │ │ │ │ ├── events.js │ │ │ │ │ ├── find.js │ │ │ │ │ ├── import.js │ │ │ │ │ ├── init.js │ │ │ │ │ ├── modals.js │ │ │ │ │ ├── tooltips.js │ │ │ │ │ └── ui.js │ │ │ │ ├── global/ │ │ │ │ │ ├── activities.js │ │ │ │ │ ├── eventlog.js │ │ │ │ │ ├── global.js │ │ │ │ │ ├── helpers.js │ │ │ │ │ └── silly.js │ │ │ │ ├── index.css │ │ │ │ ├── index.html │ │ │ │ ├── index.js │ │ │ │ ├── index.txt │ │ │ │ ├── message/ │ │ │ │ │ ├── events.js │ │ │ │ │ ├── html-sandbox.js │ │ │ │ │ ├── init.js │ │ │ │ │ ├── message.js │ │ │ │ │ ├── tooltips.js │ │ │ │ │ └── ui.js │ │ │ │ ├── pwa/ │ │ │ │ │ ├── manifest.json │ │ │ │ │ └── service-worker.js │ │ │ │ ├── search/ │ │ │ │ │ ├── bulk_actions.js │ │ │ │ │ ├── display_modes.js │ │ │ │ │ ├── events.js │ │ │ │ │ ├── init.js │ │ │ │ │ ├── selection_actions.js │ │ │ │ │ ├── tooltips.js │ │ │ │ │ └── ui.js │ │ │ │ ├── settings/ │ │ │ │ │ └── content.js │ │ │ │ ├── tags/ │ │ │ │ │ ├── events.js │ │ │ │ │ ├── init.js │ │ │ │ │ └── modals.js │ │ │ │ ├── templates/ │ │ │ │ │ ├── modal-add-tag.html │ │ │ │ │ ├── modal-auto.html │ │ │ │ │ ├── modal-compose-quoted-reply.html │ │ │ │ │ ├── modal-composer-encryption-helper.html │ │ │ │ │ ├── modal-contact-add.html │ │ │ │ │ ├── modal-display-keybindings.html │ │ │ │ │ ├── modal-related-search.html │ │ │ │ │ ├── modal-save-search.html │ │ │ │ │ ├── modal-search-keyservers.html │ │ │ │ │ ├── modal-send-public-key.html │ │ │ │ │ ├── modal-tag-picker.html │ │ │ │ │ ├── modal-upload-key.html │ │ │ │ │ └── template-modal-tag-picker-item.html │ │ │ │ └── ui/ │ │ │ │ ├── content.js │ │ │ │ ├── events.js │ │ │ │ ├── global.js │ │ │ │ ├── init.js │ │ │ │ ├── keybindings.js │ │ │ │ ├── notifications.js │ │ │ │ ├── selection.js │ │ │ │ ├── sidebar.js │ │ │ │ ├── tagging.js │ │ │ │ ├── terminal.js │ │ │ │ ├── tooltips.js │ │ │ │ └── topbar.js │ │ │ ├── layouts/ │ │ │ │ ├── auth.html │ │ │ │ ├── content-tall.html │ │ │ │ ├── content-wide.html │ │ │ │ ├── content.html │ │ │ │ ├── full-tall.html │ │ │ │ ├── full-wide.html │ │ │ │ ├── full.html │ │ │ │ ├── minimal-tall.html │ │ │ │ ├── minimal-wide.html │ │ │ │ └── minimal.html │ │ │ ├── logs/ │ │ │ │ ├── events/ │ │ │ │ │ └── index.html │ │ │ │ ├── layout.html │ │ │ │ └── network/ │ │ │ │ └── index.html │ │ │ ├── message/ │ │ │ │ ├── compose/ │ │ │ │ │ └── index.html │ │ │ │ ├── download/ │ │ │ │ │ └── index.html │ │ │ │ ├── draft/ │ │ │ │ │ └── index.html │ │ │ │ ├── forward/ │ │ │ │ │ └── forward.html │ │ │ │ ├── index.html │ │ │ │ ├── reply/ │ │ │ │ │ ├── composer.html │ │ │ │ │ └── index.html │ │ │ │ ├── send/ │ │ │ │ │ └── index.html │ │ │ │ ├── single.html │ │ │ │ └── update/ │ │ │ │ └── index.html │ │ │ ├── page/ │ │ │ │ ├── contribute/ │ │ │ │ │ ├── bugs.html │ │ │ │ │ └── index.html │ │ │ │ ├── entropy/ │ │ │ │ │ └── index.html │ │ │ │ ├── gmail-2-step-verification/ │ │ │ │ │ └── index.html │ │ │ │ ├── gmail-access-non-google-accounts/ │ │ │ │ │ └── index.html │ │ │ │ ├── index.html │ │ │ │ ├── multipile/ │ │ │ │ │ └── index.html │ │ │ │ ├── release-notes/ │ │ │ │ │ ├── credits-code.html │ │ │ │ │ ├── credits-i18n.html │ │ │ │ │ ├── credits.html │ │ │ │ │ ├── index.html │ │ │ │ │ └── license.html │ │ │ │ └── template-straw-man/ │ │ │ │ ├── index.html │ │ │ │ ├── index.js │ │ │ │ └── tags.html │ │ │ ├── partials/ │ │ │ │ ├── compose.html │ │ │ │ ├── error_message_missing.html │ │ │ │ ├── errors_content.html │ │ │ │ ├── head.html │ │ │ │ ├── helpers.html │ │ │ │ ├── hidden.html │ │ │ │ ├── javascript.html │ │ │ │ ├── modals.html │ │ │ │ ├── pile_compose.html │ │ │ │ ├── pile_email_hints.html │ │ │ │ ├── pile_email_tags.html │ │ │ │ ├── pile_message.html │ │ │ │ ├── pile_threading.html │ │ │ │ ├── search_item.html │ │ │ │ ├── sidebar.html │ │ │ │ ├── tag_add.html │ │ │ │ ├── thread_message_cryptofail.html │ │ │ │ ├── tools_contacts.html │ │ │ │ ├── tools_default.html │ │ │ │ ├── tools_message.html │ │ │ │ ├── tools_search.html │ │ │ │ ├── tools_settings.html │ │ │ │ ├── tools_tags.html │ │ │ │ ├── tooltips.html │ │ │ │ └── topbar.html │ │ │ ├── plugins/ │ │ │ │ └── index.html │ │ │ ├── profiles/ │ │ │ │ ├── account-form.html │ │ │ │ ├── add/ │ │ │ │ │ └── index.html │ │ │ │ ├── edit/ │ │ │ │ │ └── index.html │ │ │ │ ├── index.html │ │ │ │ └── remove/ │ │ │ │ └── index.html │ │ │ ├── quitquitquit/ │ │ │ │ └── index.html │ │ │ ├── search/ │ │ │ │ ├── atts.html │ │ │ │ ├── default.html │ │ │ │ ├── drafts.html │ │ │ │ ├── index.html │ │ │ │ ├── outbox.html │ │ │ │ ├── outgoing.html │ │ │ │ ├── photos.html │ │ │ │ ├── prev_more_next.html │ │ │ │ ├── sent.html │ │ │ │ └── trash.html │ │ │ ├── settings/ │ │ │ │ ├── index.html │ │ │ │ ├── mailbox/ │ │ │ │ │ └── index.html │ │ │ │ ├── plugins.html │ │ │ │ ├── preferences.html │ │ │ │ ├── privacy.html │ │ │ │ ├── recipes.html │ │ │ │ └── set/ │ │ │ │ ├── index.html │ │ │ │ └── password/ │ │ │ │ ├── index.html │ │ │ │ └── keys.html │ │ │ ├── setup/ │ │ │ │ ├── oauth2/ │ │ │ │ │ └── index.html │ │ │ │ ├── password/ │ │ │ │ │ └── index.html │ │ │ │ └── welcome/ │ │ │ │ └── index.html │ │ │ └── tags/ │ │ │ ├── add/ │ │ │ │ └── index.html │ │ │ ├── edit.html │ │ │ ├── form.html │ │ │ ├── index.html │ │ │ └── sidebar.html │ │ ├── index.html │ │ ├── js/ │ │ │ ├── helpers.js │ │ │ ├── libraries.js │ │ │ └── mousetrap.global.bind.js │ │ ├── less/ │ │ │ ├── app/ │ │ │ │ ├── attachments.less │ │ │ │ ├── compose.less │ │ │ │ ├── contacts.less │ │ │ │ ├── crypto.less │ │ │ │ ├── files.less │ │ │ │ ├── global.less │ │ │ │ ├── helpers.less │ │ │ │ ├── icons.less │ │ │ │ ├── library-override.less │ │ │ │ ├── login.less │ │ │ │ ├── message.less │ │ │ │ ├── mobile.less │ │ │ │ ├── modals.less │ │ │ │ ├── navigation.less │ │ │ │ ├── notifications.less │ │ │ │ ├── pile.less │ │ │ │ ├── profiles.less │ │ │ │ ├── screens.less │ │ │ │ ├── search.less │ │ │ │ ├── settings.less │ │ │ │ ├── setup.less │ │ │ │ ├── sidebar.less │ │ │ │ ├── tablet.less │ │ │ │ ├── tags.less │ │ │ │ ├── terminal.less │ │ │ │ ├── thread.less │ │ │ │ ├── tooltips.less │ │ │ │ ├── topbar.less │ │ │ │ └── webfonts.less │ │ │ ├── config.less │ │ │ ├── default.less │ │ │ ├── libraries/ │ │ │ │ ├── bootstrap.less │ │ │ │ ├── dropdowns.less │ │ │ │ ├── modals.less │ │ │ │ └── typeahead.less │ │ │ └── print.less │ │ ├── theme.json │ │ └── webfonts/ │ │ ├── LICENSE │ │ └── index.html │ ├── locale/ │ │ ├── README.md │ │ └── mailpile.pot │ ├── mailpile-gui/ │ │ ├── icons-osx/ │ │ │ └── mk_icons.sh │ │ ├── mailpile-gui.py │ │ ├── mailpile.desktop │ │ └── media/ │ │ └── background.xcf │ └── multipile/ │ ├── README.md │ ├── mailpile-admin.py │ ├── mailpile-launcher.py │ ├── multipile.rc.sample │ └── www/ │ ├── apache-broken.html │ ├── index.html │ └── not-running.html ├── test-requirements.txt └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ # ========================================================================= # COVERAGE CONFIGURATION FILE: .coveragerc # ========================================================================= # LANGUAGE: Python # SEE ALSO: # * http://nedbatchelder.com/code/coverage/ # * http://nedbatchelder.com/code/coverage/config.html # ========================================================================= [coverage:run] append = .coverage include = mailpile* omit = nose branch = True #parallel = True [coverage:report] # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma pragma: no cover # Don't complain about missing debug-only code: def __repr__ if self\.debug # Don't complain if tests don't hit defensive assertion code: raise AssertionError raise NotImplementedError # Don't complain if non-runnable code isn't run: if 0: if False: if __name__ == .__main__.: ignore_errors = True [coverage:html] directory = build/coverage.html [coverage:xml] outfile = build/coverage.xml ================================================ FILE: .dockerignore ================================================ Dockerfile Vagrantfile ================================================ FILE: .gitignore ================================================ *.pyc *.pyo *~ *.po *.debhelper *.iml *.ipr *.swp *.swo mp-virtualenv .idea .*deps .tox mailpile.egg-info/ mailpile-tmp.py static/default/plugins/ mailpile/tests/data/private-test-data.mbx mailpile/tests/data/tmp mailpile/tests/data/gpg-keyring/random_seed scripts/less-compiler.mk .DS_Store .vagrant .coverage ghostdriver.log build/ dist/ .eggs/ cover/ bower_components/ node_modules/ testing/ mailpile.tar.gz AUTHORS ChangeLog shared-data/multipile/www/admin.cgi setup-tmp/ ================================================ FILE: .gitmodules ================================================ [submodule "doc"] path = doc url = https://github.com/pagekite/Mailpile.wiki.git branch = master [submodule "shared-data/contrib/print"] path = shared-data/contrib/print url = https://github.com/mailpile/Mailpile-print.git [submodule "submodules/gui-o-matic"] path = submodules/gui-o-matic url = https://github.com/mailpile/gui-o-matic [submodule "submodules/gui-o-mac-tic"] path = submodules/gui-o-mac-tic url = https://github.com/mailpile/gui-o-mac-tic ================================================ FILE: .travis.yml ================================================ language: python python: 2.7 env: - TOX_ENV=py27 addons: apt: packages: - gnupg install: - pip install tox coveralls script: - tox -e $TOX_ENV after_success: - coveralls ================================================ FILE: .tx/config ================================================ [main] host = https://www.transifex.com [mailpile.mailpilepot] source_file = shared-data/locale/mailpile.pot source_lang = en type = PO file_filter = shared-data/locale//LC_MESSAGES/mailpile.po ================================================ FILE: AGPLv3.txt ================================================ NOTE: Please see the file COPYING.md for details on Mailpile licensing. GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # The Mailpile Code of Conduct The Mailpile project depends on its community of volunteers, supporters and of course users. We want people to feel welcome and respected. To this end, project maintainers have plegded to adhere to and enforce the Contrbitor Covenant Code of Conduct, in their Mailpile related activities, both online and in person. We respectfully ask that other contributors and participants in our community do so as well. Thank you for your understanding, and welcome! # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ ================================================ FILE: CONTRIBUTING.md ================================================ ## How to contribute to Mailpile First of all: Thank you! :heart: #### Further Reading Second of all, please adhere to our Code of Conduct when you participate in our community. Be kind, be respectful. [The full Code of Conduct can be found here](CODE_OF_CONDUCT.md). Please also be sure you are comfortable with [our license](COPYING.md) before contributing any code to Mailpile. Note that Mailpile is a slightly unsual app; the design is both political and opinionated. If you are confused or unsure whether something is a bug or a feature, [our developer FAQ might have answers](DEV_FAQ.md)! Please read it before filing issues about design choices or attempting to reorganize the code in any major way. The FAQ also contains a quick introduction Mailpile internals and debugging techniques. Also, have you seen [the main Mailpile website](https://www.mailpile.is/) and [the community Discourse](https://community.mailpile.is/categories)? #### Are you new to Mailpile and/or FOSS? * Welcome! The first step is probably to get Mailpile up and running, play with it, get familiar. * Read the links above, and [check out our website](https://www.mailpile.is/) if you haven't already. * Not sure what to work on? Find inspiration in one of our [Low Hanging Fruit](https://github.com/mailpile/Mailpile/issues?q=is%3Aissue+is%3Aopen+label%3A%22Low+Hanging+Fruit%22) issues! * Otherwise, read on... #### Did you find a bug? * **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/mailpile/Mailpile/issues). * If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/mailpile/Mailpile/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible. * Remember, we cannot read your mind or see your screen, so you'll probably need to answer all of these: 1. What were you doing? 2. What did you expect would happen? 3. What actually happened? 4. Operating system? GnuPG version? Mailpile version? * Reproducability is key. If you cannot reliably trigger the bug or cannot describe how to do so, then unfortunately it's less likely that the Mailpile team will be able to do anything about it. It may still be useful to file a report in case others are having the same issue, but bugs that can be reproduced will in general get fixed much faster! * The [Developer FAQ](DEV_FAQ.md) has a section on debugging techniques. * If it's not bug, but you still need help: [visit our Discourse support forum](https://community.mailpile.is/c/support). #### Did you write a patch that fixes a bug? * Open a new GitHub pull request with the patch. * Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. #### Do you plan write a patch that adds a feature? * Please be sure you have read the [Developer FAQ](DEV_FAQ.md) to ensure your feature is compatible with our high-level goals and design decisions. * If you are unsure or would like some guidance, join the #mailpile channel on IRC (Freenode) and discuss your plans there. * Open a new GitHub pull request with the patch. * Ensure the PR description clearly describes what your feature does. Include the relevant issue number if applicable. #### Did you fix whitespace, format code, or make a purely cosmetic patch? Changes that are cosmetic in nature and do not add anything substantial to the stability, functionality, or testability of Mailpile will generally not be accepted. All patches to Mailpile are reviewed by a human and our resources (time, people) are very limited. Cosmetic patches are no easier to review than other patches and we would rather focus our efforts on functional improvements to the software. #### Do you want help other users? Our [community Discourse](https://community.mailpile.is/categories) has a support forum and needs friendly people to help out and steer conversations in friendly and productive directions. Please join the conversation! #### Do you want to translate Mailpile to your language? Please feel free to join our translation community [on Transifex](https://www.transifex.com/otf/mailpile/). #### Do you plan to write documentation? * Please feel free to contribute documentation to [our wiki](https://github.com/mailpile/Mailpile/wiki). * If you would like some guidance, join the #mailpile channel on IRC (Freenode) or file an issue with your questions. * If you added a new page or made major major changes to an existing page, please file [a new issue requesting a review](https://github.com/mailpile/Mailpile/issues/new) when you are done. Be sure to include a link to the wiki page! #### Credits This document borrows some language and structure from the [Ruby on Rails contributor guidelines](https://raw.githubusercontent.com/rails/rails/master/CONTRIBUTING.md). ================================================ FILE: COPYING.md ================================================ # Mailpile - a program for doing stuff with e-mail Copyright (C) 2011-2015, Bjarni R. Einarsson, Mailpile ehf and friends. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . ## What happened to the Apache license? This software used to be dual-licensed under AGPLv3 and the Apache License 2.0. If you feel an uncontrollable urge to fork and carry on without the AGPL, the last dual-licensed code point is tagged as "Gunsmoke--TheLastApacheTag". Have fun! For more details on our licensing choices, see these blog posts: * * * Thanks! ================================================ FILE: DEV_FAQ.md ================================================ ## Mailpile Developer FAQ This document contain a collection of frequently asked questions (with answers) about Mailpile development. Please familiarize yourself with the contents before attempting any deep hacking on Mailpile. You don't have to agree with all of our priorities to take part or make use of Mailpile, but we do feel it helps if most of the community is rowing in roughly the same general direction! Note: If you are just looking for debugging tips and tricks, you can skip to the end. ### Why Mailpile? The long-term goal of Mailpile is to help *non-technical people* become *more independent* and *more private* online, in particular when it comes to e-mail. By **more independent**, we mean people should be in control of their e-mail and the software used to manage it. This is why Mailpile is Free Software, and this is why we *don't* promote Mailpile as a tool for building "cloud services". By **more private**, we mean people should have more control over who has access to their e-mail, when and how. This is why Mailpile tries to make e-mail encryption more accessible, and this is why Mailpile tries to make it convenient *and secure* for people to store their e-mail on their own devices. Our focus on **non-technical people** implies, amongst other things, that we cannot exclude people who are using non-free platforms such as Microsoft Windows or Mac OS X, and we cannot require our users learn a radically new, unfamiliar user interface or understand highly technical concepts such as public key cryptography. ### Are All of These Decisions Awesome? No. Some of these decisions were almost certainly mistakes. But that's part of innovating, you try new things and see how they go! So we kindly request that our contributors and collaboratores refrain from picking arguments with us over these decisions. Making forward progress is much more fun than rehashing the past over and over again. ### Why Do You Reinvent So Many Wheels? Usually, the answer is one or more of the following: * We didn't know a solution already existed * We evaluated solutions X, Y and Z, but didn't like them * We felt the problem was simple enough to Just Do It In general, we are reluctant to take on new external dependencies unless they are stable and widely available: if they exist in pre-packaged form for Windows, Mac OS X and the most common Linux distributions, they are probably fine! If not, the benefits need to outweigh the added complexity posed for our cross-platform packaging efforts. For this reason we also tend to prefer pure-Python (or Javascript, for the UI) libraries over native code, which also avoids certain classes of bugs and security vulnerabilities which are simply not present in a memory-safe, managed language. We have broken this rule more often than we'd like in our user-interface code, due to cultural differences between web developers and Python folks. *We would like to gradually reduce our front-end dependencies.* ### Why a Search Engine? Mailpile started as an experimental search engine, a hobby project. Everything else came later. *OK, so why not replace it with a standard component?* Because it works! Replacing it at this point would be *more work, not less*. Also, we are unaware of any "off the shelf" search engines that let us encrypt their data stores and we feel encrypting the search index is an absolute requirement since we by default allow the user to search inside encrypted messages. *I see. Why not remove it, to simplify the code?* The entire app is built around the search metaphor, much like Google's GMail. This is a fundamentally different way to build an e-mail client, from the traditional "messages and mailboxes" model. The search engine also makes it easy for Mailpile to do some pretty cool things. For example: * Mailpile can evaluate the trustworthiness of a message by asking the search engine about the past behaviour of the sender * Mailpile can postpone processing of things like attached PGP keys until it actually intends to use the key: the search engine makes the keys easy to find later on ### Why doesn't Mailpile have a Native User Interface? For now, because the team is very small and we would like to reach users on many different platforms, we have bet entirely on the web as our primary user interface: * This decision allows us to target any operating system for which Python 2.7 has been made available. * It also allows us to get help from the enormous community of web developers; this is a much larger talent pool than the pool of developers that know how to write native apps. * Last but not least, the web interface is key to our plan to allow users to access their Mailpile remotely over the network. Remote access is critical if we want to get people to store their e-mail on their own devices, because most people read their mail on multiple devices (laptop, tablet, mobile, work computer, etc). That said: we do want to have a minimal native user interface, on all the major desktop operating systems. [That is what the gui-o-matic spin-off project is about](https://github.com/mailpile/gui-o-matic). Native mobile apps on Android and/or iOS would also be nice to have! ### How Does Mailpile Handle Security? This is a huge topic! Please consult our [Security Roadmap](https://github.com/mailpile/Mailpile/wiki/Security-roadmap). Security is complex and means different things to different people. The most controversial questions relating to security, have to do with mass surveillance and law enforcement. Mailpile's stance is that we believe that people have an innate right to privacy we believe that mass surveillance is *wrong*. If the government wants to read your e-mail, we feel they should present you with a search warrant. Thwarting other adversaries (criminals, jealous partners, etc.) is also very much something we care about, but is probably less divisive. ### Why GnuPG? Why not GPGME? Why not PGPy? GnuPG is mature and stable. Although the user interfaces may leave something to be desired, it has a rich ecosystem of powerful tools built around it and a wealth of documentation and support to be found online. If we didn't use GnuPG, we would have to reinvent a lot of wheels that aren't central to Mailpile's mission. Our issue tracker contains [further discussions on the topic of why GnuPG and not something else, such as PGPy](https://github.com/mailpile/Mailpile/issues/1743). [Use of GPGME is also being discussed](https://github.com/mailpile/Mailpile/issues/1742). Currently, we don't feel GPGME provides enough benefit to justify the additional dependency and the additional (hypothetical) risk posed by solving the GnuPG integration problem with a large amount of code written in C (as opposed to Python, which is memory-safe and less likely to contain certain classes of security vulnerabilities). ### Why not Python 3? We depend on some libraries - spambayes in particular - which were not available for Python 3 when we started this project. We don't think Python 2 is going away in the near future. [There is a Github issue discussing this.](https://github.com/mailpile/Mailpile/issues/160) ### Why Not Django? Or Flask? Reasons! Our way may be unusual, but it's kinda awesome once you get used to it and it wasn't obvious to us how we could get this kind of behaviour from one of the standard frameworks. Please read the next section for details. ### How Does Mailpile's Web UI Work? The text-based command-line interface is an important part of Mailpile's user interface. Our home-brewed framework allows us to generate web API end-points, text commands and command-line arguments *at the same time*. Our internal framework also has the concept of commands supporting multiple output formats; so the same API endpoint can generate text, templated HTML, JSON and XML-RPC interfaces with relatively little additional code. Some endpoints also generate XML, CSV, CSS or Javascript. You can try this yourself, simply by editing the URL in your browser: # The default, rendered as HTML http://localhost:33411/in/inbox/ http://localhost:33411/in/inbox/as.html http://localhost:33411/search/?q=in:inbox # Same thing, as JSON http://localhost:33411/in/inbox/as.json http://localhost:33411/api/0/search/?q=in:inbox # Same thing, as text http://localhost:33411/in/inbox/as.text http://localhost:33411/api/0/search/as.text?q=in:inbox The filename part of the URL is used to select output formats. All endpoints support `as.json`, most support HTML and/or text. The HTML output of each command is generated using Jinja2 templates that are found in `shared-data/default-theme/html/...`. The directory structure generally matches the URL paths seen in the browser, with the main template for each command named `index.html`. Alternate templates for the same API endpoint can have other names, for example a template named `.../html/search/social.html` would be accessible using URLs like so: http://localhost:33411/in/inbox/social.html http://localhost:33411/search/social.html?q=from:person@foo.com ### Can I Develop Plugins For Mailpile? Sort-of! Internally, the app is quite modular and there are methods which allow code to register classes or functions that perform various functions. However, the plugin API is not considered stable, it is incomplete and it is not very well documented. It may also not be a very nice API, and we rather expect it to develop and change rapidly post-1.0. If you are interested in Mailpile's plugin APIs, take a look in `shared-data/contrib/` for some examples of "external plugins" and `mailpile/plugins/` for "internal plugins". ### How Do I Debug Mailpile? Developers should learn to use the Mailpile CLI. The `mailpile>` prompt is where all of the low-level magic happens. Future versions of Mailpile will expose this functionality to the web interface itself, but for now you will need to use your shell. Possibly the most important command for Mailpile hackers, is to know how to enable debugging. An example: # Enable verbose debugging of HTTP requests and GnuPG integration # Note: HTTP debugging disables all sorts of internal caches! mailpile> set sys.debug = log http gnupg ... Many other subsystems can have debugging enabled. At the time of writing, the `sys.debug` can include the following terms to make various parts of the app more verbose: log http compose cryptostate autotag rescan keywords cache connbroker vcard pop3 gnupg keylookup imap sources jinja timing sendmail httpdata There are also a few other ways to examine the app state: # Watch logging and debug messages fly by mailpile> eventlog/watch ... [CTRL+C] ... # Examine event log (piped through less) mailpile> pipe less eventlog ... mailpile> pipe less eventlog incomplete ... # Get an overview of what threads are running and what they are doing mailpile> ps ... Low-level changes and exploration of the configuration are also best done from the CLI: # Explore the configuration; see also mailpile/config/defaults.py mailpile> print -short sys ... mailpile> print -flat sources ... mailpile> print -secrets secrets ... # Change things (dangerous) mailpile> set sys.gpg_binary = /bin/false ... # Reset something to its default setting mailpile> unset sys.gpg_binary ... There is also a help command, and you can use tab completion to try and "guess" what commands exist. mailpile> help ... mailpile> help tags ... Finally, the app ships with a `hacks` plugin which is disabled by default. If you load it, that will add a few more low-level commands, including an embedded Python interpretor: mailpile> plugins/load hacks ... mailpile> hacks/pycli ... There's sure to be more; please feel free to file a pull request against this document to add your favourite tricks or clarify these. ================================================ FILE: Dockerfile ================================================ FROM debian:stretch-slim ENV GID 33411 ENV UID 33411 RUN apt-get update && \ apt-get install -y curl apt-transport-https gnupg && \ curl -s https://packages.mailpile.is/deb/key.asc | apt-key add - && \ echo "deb https://packages.mailpile.is/deb release main" | tee /etc/apt/sources.list.d/000-mailpile.list && \ apt-get update && \ apt-get install -y mailpile && \ # TODO Enable apache for multi users # apt-get install -y mailpile-apache2 update-rc.d tor defaults && \ service tor start && \ groupadd -g $GID mailpile && \ useradd -u $UID -g $GID -m mailpile && \ su - mailpile -c 'mailpile setup' && \ apt-get clean WORKDIR /home/mailpile USER mailpile VOLUME /home/mailpile/.local/share/Mailpile EXPOSE 33411 CMD mailpile --www=0.0.0.0:33411/ --wait ================================================ FILE: Dockerfile.dev ================================================ FROM mailpile # Install C compiler for python deps w/ native extensions RUN apk --no-cache add \ gcc \ libc-dev \ python-dev \ shadow # Workaround: Setting mailpile users uid to 1000 to have write permissions # from the docker host the to the shared volumes /Mailpile and /mailpile-data. # Mounted volumes seem to be configured w/ uid/guid = 1000. # Learn more here: # - https://github.com/docker/docker/issues/7198 # - https://denibertovic.com/posts/handling-permissions-with-docker-volumes/ RUN usermod -u 1000 mailpile RUN pip install -r requirements-dev.txt RUN chmod +x /entrypoint.sh CMD ["./mp"] ================================================ FILE: Gruntfile.js ================================================ module.exports = function(grunt) { grunt.registerTask('watch', [ 'watch' ]); grunt.initConfig({ concat: { js: { options: { separator: ';' }, src: [ 'bower_components/jquery/dist/jquery.min.js', 'bower_components/underscore/underscore-min.js', 'bower_components/jquery-timer/jquery.timer.js', 'bower_components/autosize/dist/autosize.js', 'bower_components/mousetrap/mousetrap.js', 'shared-data/default-theme/js/mousetrap.global.bind.js', 'bower_components/jquery.ui/ui/jquery.ui.core.js', 'bower_components/jquery.ui/ui/jquery.ui.widget.js', 'bower_components/jquery.ui/ui/jquery.ui.mouse.js', 'bower_components/jquery.ui/ui/jquery.ui.draggable.js', 'bower_components/jquery.ui/ui/jquery.ui.droppable.js', 'bower_components/jquery.ui/ui/jquery.ui.sortable.js', 'bower_components/jqueryui-touch-punch/jquery.ui.touch-punch.js', 'bower_components/qtip2/basic/jquery.qtip.min.js', 'bower_components/jquery-slugify/dist/slugify.js', 'bower_components/typeahead.js/dist/typeahead.jquery.js', 'bower_components/bootstrap/js/dropdown.js', 'bower_components/bootstrap/js/modal.js', 'bower_components/favico.js/favico.js', 'bower_components/select2/select2.min.js', 'bower_components/moxie/bin/js/moxie.min.js', 'bower_components/plupload/js/plupload.min.js', 'bower_components/dompurify/dist/purify.min.js' ], dest: 'shared-data/default-theme/js/libraries.min.js' } }, uglify: { options: { mangle: false }, js: { files: { 'shared-data/default-theme/js/libraries.min.js': ['shared-data/default-theme/js/libraries.min.js'] } } }, less: { options: { cleancss: true }, style: { files: { "shared-data/default-theme/css/default.css": "shared-data/default-theme/less/default.less" } } }, watch: { js: { files: ['shared-data/default-theme/js/*.js'], tasks: ['concat:js', 'uglify:js'], options: { livereload: true, } }, css: { files: [ 'shared-data/default-theme/less/config.less', 'shared-data/default-theme/less/default.less', 'shared-data/default-theme/less/app/*.less', 'shared-data/default-theme/less/libraries/*.less' ], tasks: ['less:style'], options: { livereload: true, } } } }); grunt.loadNpmTasks('grunt-contrib-concat'); grunt.loadNpmTasks('grunt-contrib-uglify'); grunt.loadNpmTasks('grunt-contrib-less'); grunt.loadNpmTasks('grunt-contrib-watch'); }; ================================================ FILE: MANIFEST.in ================================================ recursive-exclude locale * recursive-exclude scripts * recursive-exclude testing * recursive-include mailpile/tests/data * recursive-exclude mailpile/tests/data *~ recursive-include mailpile/locale *.mo *.po *.pot recursive-include mailpile/plugins * include Makefile include README.md include AGPLv3.txt include LICENSE-2.0.txt include requirements.txt include test-requirements.txt include requirements-dev.txt include install_hooks.py include scripts/compile-messages.sh include scripts/make-messages.sh include scripts/mailpile-test.py include scripts/test-sendmail.sh include scripts/setup-test.sh recursive-include shared-data * ================================================ FILE: Makefile ================================================ # Recipes for stuff export PYTHONPATH := . help: @echo "" @echo "BUILD" @echo " dpkg" @echo " Create a debian package of this service (in a Docker " @echo " container)." @echo "" all: submodules alltests docs web compilemessages transifex dev: @echo export PYTHONPATH=`pwd` arch-dev: sudo pacman -Syu --needed community/python2-pillow extra/python2-lxml community/python2-jinja \ community/python2-pep8 extra/python2-nose community/phantomjs \ extra/python2-pip community/python2-mock \ extra/ruby community/npm community/spambayes TMPDIR=`mktemp -d /tmp/aur.XXXXXXXXXX`; \ cd $$TMPDIR; \ pacman -Qs '^yuicompressor$$' > /dev/null; \ if [ $$? -ne 0 ]; then \ sudo pacman -S --needed core/base-devel; \ curl -s https://aur.archlinux.org/cgit/aur.git/snapshot/yuicompressor.tar.gz | tar xzv; \ cd yuicompressor; \ makepkg -si; \ cd $$TMPDIR; \ fi; \ cd /tmp; \ rm -rf $$TMPDIR sudo pip2 install 'selenium>=2.40.0' which lessc >/dev/null || sudo gem install therubyracer less which bower >/dev/null || sudo npm install -g bower which uglify >/dev/null || sudo npm install -g uglify fedora-dev: sudo yum install python-imaging python-lxml python-jinja2 python-pep8 \ ruby ruby-devel python-yui python-nose spambayes \ phantomjs python-pip python-mock npm sudo yum install rubygems; \ sudo yum install python-pgpdump || pip install pgpdump sudo pip install 'selenium>=2.40.0' which lessc >/dev/null || sudo gem install therubyracer less which bower >/dev/null || sudo npm install -g bower which uglify >/dev/null || sudo npm install -g uglify debian-dev: sudo apt-get install python-imaging python-lxml python-jinja2 pep8 \ ruby-dev yui-compressor python-nose spambayes \ python-pip python-mock python-selenium \ rubygems-integration dpkg -l|grep -qP ' nodejs .*nodesource' || sudo apt install npm sudo apt-get install python-pgpdump || pip install pgpdump which phantomjs >/dev/null || sudo apt-get install phantomjs || sudo npm install -g phantomjs which lessc >/dev/null || sudo gem install therubyracer less which bower >/dev/null || sudo npm install -g bower which uglify >/dev/null || sudo npm install -g uglify submodules: git submodule update --remote docs: submodules @python2.7 mailpile/urlmap.py |grep -v ^FIXME: >doc/URLS.md @ls -l doc/URLS.md @python2.7 mailpile/defaults.py |grep -v -e ^FIXME -e ';timestamp' \ >doc/defaults.cfg @ls -l doc/defaults.cfg web: less js @true alltests: clean pytests @chmod go-rwx mailpile/tests/data/gpg-keyring @DISPLAY= nosetests @DISPLAY= python2.7 scripts/mailpile-test.py || true @git checkout mailpile/tests/data/ pytests: @echo -n 'security ' && python2.7 mailpile/security.py @echo -n 'urlmap ' && python2.7 mailpile/urlmap.py -nomap @echo -n 'search ' && python2.7 mailpile/search.py @echo -n 'mailboxes.mbox ' && python2.7 mailpile/mailboxes/mbox.py @echo -n 'mailutils.safe ' && python2.7 mailpile/mailutils/safe.py @echo -n 'mailutils.addrs ' && python2.7 mailpile/mailutils/addresses.py @echo -n 'mailutils.emails ' && python2.7 mailpile/mailutils/emails.py @echo -n 'config/base ' && python2.7 mailpile/config/base.py @echo -n 'config/validators' && python2.7 mailpile/config/validators.py @echo -n 'config/manager ' && python2.7 mailpile/config/manager.py @echo -n 'conn_brokers ' && python2.7 mailpile/conn_brokers.py @echo -n 'crypto/autocrypt ' && python2.7 mailpile/crypto/autocrypt.py @echo -n 'plug...autocrypt ' && python2.7 mailpile/plugins/crypto_autocrypt.py @echo -n 'crypto/mime ' && python2.7 mailpile/crypto/mime.py @echo -n 'index.base ' && python2.7 mailpile/index/base.py @echo -n 'index.msginfo ' && python2.7 mailpile/index/msginfo.py @echo -n 'index.mailboxes ' && python2.7 mailpile/index/mailboxes.py @echo -n 'index.search ' && python2.7 mailpile/index/search.py @echo -n 'util ' && python2.7 mailpile/util.py @echo -n 'vcard ' && python2.7 mailpile/vcard.py @echo -n 'workers ' && python2.7 mailpile/workers.py @echo -n 'packing ' && python2.7 mailpile/packing.py @echo -n 'mailboxes/pop3 ' && python2.7 mailpile/mailboxes/pop3.py @echo -n 'mail_source/imap ' && python2.7 mailpile/mail_source/imap.py @echo -n 'crypto/aes_utils ' && python2.7 mailpile/crypto/aes_utils.py @echo 'spambayes... ' && python2.7 mailpile/spambayes/Tester.py @echo 'crypto/streamer...' && python2.7 mailpile/crypto/streamer.py @echo clean: @rm -f `find . -name \\*.pyc` \ `find . -name \\*.pyo` \ `find . -name \\*.mo` \ mailpile-tmp.py mailpile.py \ ChangeLog AUTHORS \ .appver MANIFEST .SELF .*deps \ dist/*.tar.gz dist/*.deb dist/*.rpm \ scripts/less-compiler.mk ghostdriver.log @rm -rf *.egg-info build/ \ mailpile/tests/data/tmp/ testing/tmp/ @rm -f shared-data/multipile/www/admin.cgi mrproper: clean @rm -rf shared-data/locale/?? shared-data/locale/??[_@]* @rm -rf bower_components/ shared-data/locale/mailpile.pot @rm -rf mp-virtualenv/ git reset --hard && git clean -dfx sdist: clean @python setup.py sdist #bdist-prep: compilemessages web -- FIXME: Make building web assets work! bdist-prep: compilemessages @true bdist: @python setup.py bdist_wheel virtualenv: mp-virtualenv/bin/activate virtualenv-dev: mp-virtualenv/bin/.dev mp-virtualenv/bin/activate: virtualenv -p python2.7 --system-site-packages mp-virtualenv bash -c 'source mp-virtualenv/bin/activate && pip install -r requirements.txt && python setup.py install' @rm -rf mp-virtualenv/bin/.dev @echo @echo NOTE: If you want to test/develop with GnuPG 2.1, you might @echo want to activate the virtualenv and then run this script @echo to build GnuPG 2.1: ./scripts/add-gpgme-and-gnupg-to-venv @echo mp-virtualenv/bin/.dev: virtualenv rm -rf mp-virtualenv/lib/python2.7/site-packages/mailpile cd mp-virtualenv/lib/python2.7/site-packages/ && ln -s ../../../../mailpile rm -rf mp-virtualenv/share/mailpile cd mp-virtualenv/share/ && ln -s ../../shared-data mailpile @touch mp-virtualenv/bin/.dev bower_components: @bower install js: bower_components # Warning: Horrible hack to extract rules from Gruntfile.js @rm -f shared-data/default-theme/js/libraries.min.js @rm -f shared-data/default-theme/js/mailpile-min.js.tmp* @cat Gruntfile.js \ |sed -e '1,/concat:/d ' \ |sed -e '1,/src:/d' -e '/dest:/,$$d' \ |grep / \ |sed -e "s/[',]/ /g" \ |xargs sed -e '$$a;' \ >> shared-data/default-theme/js/mailpile-min.js.tmp @uglify -s shared-data/default-theme/js/mailpile-min.js.tmp \ -o shared-data/default-theme/js/mailpile-min.js.tmp2 @sed -e "s/@MP_JSBUILD_INFO@/`./scripts/gitwhere.sh`/" \ < shared-data/default-theme/js/libraries.js \ > shared-data/default-theme/js/libraries.min.js @echo '/* Sources...' \ >> shared-data/default-theme/js/libraries.min.js @bower --offline --no-color list \ >> shared-data/default-theme/js/libraries.min.js @echo '*/' \ >> shared-data/default-theme/js/libraries.min.js @cat shared-data/default-theme/js/mailpile-min.js.tmp2 \ >> shared-data/default-theme/js/libraries.min.js @rm -f shared-data/default-theme/js/mailpile-min.js.tmp* less: less-compiler bower_components @cp -fa \ bower_components/select2/select2.png \ bower_components/select2/select2x2.png \ bower_components/select2/select2-spinner.gif \ shared-data/default-theme/css/ @make -s -f scripts/less-compiler.mk less-loop: less-compiler @echo 'Running less compiler every 15 seconds. CTRL+C quits.' @while [ 1 ]; do \ make -s less; \ sleep 15; \ done less-compiler: bower install @cp scripts/less-compiler.in scripts/less-compiler.mk @find shared-data/default-theme/less/ -name '*.less' \ |perl -npe s'/^/\t/' \ |perl -npe 's/$$/\\/' \ >>scripts/less-compiler.mk @echo >> scripts/less-compiler.mk @perl -e 'print "\t\@touch .less-deps", $/' >> scripts/less-compiler.mk genmessages: @scripts/make-messages.sh compilemessages: @scripts/compile-messages.sh transifex: tx pull -a --minimum-perc=25 tx pull -l is,en_GB # ----------------------------------------------------------------------------- # BUILD # ----------------------------------------------------------------------------- dist/version.txt: mailpile/config/defaults.py scripts/version.py mkdir -p dist scripts/version.py > dist/version.txt dist/mailpile.tar.gz: mrproper genmessages transifex dist/version.txt git submodule update --init --recursive git submodule foreach 'git reset --hard && git clean -dfx' mkdir -p dist scripts/version.py > dist/version.txt tar --exclude='./packages/debian' --exclude=dist --exclude-vcs -czf dist/mailpile-$$(cat dist/version.txt).tar.gz -C $(shell pwd) . (cd dist; ln -fs mailpile-$$(cat version.txt).tar.gz mailpile.tar.gz) .dockerignore: dist/version.txt packages/Dockerfile_debian packages/debian packages/debian/rules mkdir -p dist docker build \ --file=packages/Dockerfile_debian \ --tag=mailpile-deb-builder \ ./ touch .dockerignore dpkg: dist/mailpile.tar.gz .dockerignore docker run \ --rm --volume=$$(pwd)/dist:/mnt/dist \ mailpile-deb-builder ================================================ FILE: README.md ================================================ # Welcome to Mailpile! # **IMPORTANT NOTE** Development on this codebase has halted, until the [Python3 rewrite](https://community.mailpile.is/t/a-very-uninformative-progress-update-mailpile-2/785) has completed. Apologies to those who have unanswered, out-standing pull requests and issues. 😢 Your efforts are appreciated! If you rely on this code and have your own branch which you actively maintain, let us know: we would be happy to link to it. If you need to run Mailpile v1 to access legacy data, consider using our [legacy Docker images](https://github.com/mailpile/Mailpile-v1-Docker). ------------------------------------------------------------------------ ## Introduction (Obsolete) ## Mailpile () is a modern, fast web-mail client with user-friendly encryption and privacy features. The development of Mailpile is funded by [a large community of backers](https://www.mailpile.is/#community) and all code related to the project is and will be released under an OSI approved Free Software license. Mailpile places great emphasis on providing a clean, elegant user interface and pleasant user experience. In particular, Mailpile aims to make it easy and convenient to receive and send PGP encrypted or signed e-mail. Mailpile's primary user interface is web-based, but it also has a basic command-line interface and an API for developers. Using web technology for the interface allows Mailpile to function both as a local desktop application (accessed by visiting `localhost` in the browser) or a remote web-mail on a personal server or VPS. The core of Mailpile is a fast search engine, custom written to deal with large volumes of e-mail on consumer hardware. The search engine allows e-mail to be organized using tags (similar to GMail's labels) and the application can be configured to automatically tag incoming mail either based on static rules or bayesian classifiers. ### Trying Mailpile If you need to run Mailpile v1 to access legacy data, consider using our [legacy Docker images](https://github.com/mailpile/Mailpile-v1-Docker). ## Credits and License ## Bjarni R. Einarsson () created this! If you think it's neat, you should also check out PageKite: . [Smári]() and [Brennan](https://brennannovak.com) joined the team in 2013 and made this a real project (not just a toy search engine). The original GMail team deserve a mention for their inspiring work: wishing the Free Software world had something like GMail is what motivated Bjarni to start working on Mailpile. We would also like to thank Edward Snowden for inspiring us to try and make PGP usable for journalists and everday folks! Contributors: - Bjarni R. Einarsson () - Brennan Novak () - Smari McCarthy () - Lots more, run `git shortlog -s` for a list! (Or check [GitHub](https://github.com/mailpile/Mailpile/graphs/contributors).) And of course, we couldn't do this without [our community of backers](https://www.mailpile.is/#community). This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation. See the file `COPYING.md` for details. ================================================ FILE: babel.cfg ================================================ # Extraction from Python source files [python: mailpile/**.py] encoding = utf-8 [python: shared-data/contrib/**.py] encoding = utf-8 [python: shared-data/multiple/**.py] encoding = utf-8 [python: shared-data/mailpile-gui/**.py] encoding = utf-8 # Extraction from Jinja2 template files [jinja2: shared-data/default-theme/html/**] encoding = utf-8 ignore_tags = style extensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand silent = false [jinja2: shared-data/contrib/html/**.html] encoding = utf-8 ignore_tags = style extensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand silent = false [jinja2: shared-data/contrib/html/**.js] encoding = utf-8 ignore_tags = style extensions = jinja2.ext.do,jinja2.ext.autoescape,mailpile.www.jinjaextensions.MailpileCommand silent = false ================================================ FILE: bower.json ================================================ { "name": "mailpile", "version": "0.1.0", "homepage": "https://mailpile.is", "authors": [ "Various" ], "description": "Front end libraries and dependencies for Mailpile default theme", "main": "libraries.js", "license": "Various", "private": true, "ignore": [ "**/.*", "node_modules", "bower_components", "test", "tests" ], "dependencies": { "underscore": ">=1.7.0", "bootstrap": "~3.3.0", "favico.js": "~0.3.5", "html5shiv": "~3.7.0", "jquery": "~2.1", "autosize": "~3.0.14", "jquery-confirm": "~2.1.1", "jquery-slugify": "~1.0.3", "jquery-timer": "*", "jquery.ui": "~1.10", "less-elements-old": "~1.0", "mousetrap": "~1.6.0", "moxie": "~1.3.4", "plupload": "~2.1.2", "qtip2": "~2.1.1", "rebar": "~0.3.2", "select2": "3.5.4", "typeahead.js": "~0.10.5", "jqueryui-touch-punch": "*", "dompurify": "~1.0.11" } } ================================================ FILE: docker-compose.dev.yml ================================================ version: '3.0' services: mailpile_dev: tty: true stdin_open: true container_name: mailpile_dev build: context: . dockerfile: Dockerfile.dev image: mailpile_dev volumes: - .:/Mailpile - .dev-mailpile-data:/mailpile-data:rw ports: - 33412:33411 ================================================ FILE: docker-compose.yml ================================================ version: '3.0' services: mailpile: container_name: mailpile build: . image: mailpile volumes: - .:/Mailpile - .dev-mailpile-data:/mailpile-data ports: - 33411:33411 ================================================ FILE: install_hooks.py ================================================ import os import sys def symlink_develop(config): if 'develop' in sys.argv: share_path = os.path.join(sys.prefix, 'share') if not os.path.exists(share_path): os.makedirs(share_path) os.symlink( os.path.join( os.path.dirname(os.path.realpath(__file__)), 'shared-data' ), os.path.join(sys.prefix, 'share', 'mailpile') ) ================================================ FILE: mailpile/__init__.py ================================================ from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n __all__ = ['Mailpile', "app", "commands", "plugins", "mailutils", "search", "ui", "util"] class Mailpile(object): """This object provides a simple Python API to Mailpile.""" def __init__(self, ui=None, workdir=None, session=None): import mailpile.app import mailpile.config.defaults import mailpile.ui if not session: ui = ui or mailpile.ui.UserInteraction self._config = mailpile.app.ConfigManager( workdir=workdir, rules=mailpile.config.defaults.CONFIG_RULES) self._session = mailpile.ui.Session(self._config) self._ui = self._session.ui = ui(self._config) self._session.config.load(self._session) self._session.main = True else: self._session = session self._config = session.config self._ui = session.ui for cls in mailpile.commands.COMMANDS: names, argspec = cls.SYNOPSIS[1:3], cls.SYNOPSIS[3] if names[0]: setattr(self, *self._mk_action(cls, names[0], argspec)) if names[1] and (names[0] != names[1]): setattr(self, *self._mk_action(cls, names[1], argspec)) def _mk_action(self, cls, cmd, argspec): import mailpile.commands if argspec: def fnc(*args, **kwargs): return mailpile.commands.Action(self._session, cmd, args, data=kwargs) else: def fnc(**kwargs): return mailpile.commands.Action(self._session, cmd, '', data=kwargs) fnc.__doc__ = '%s(%s) # %s' % (cmd, argspec or '', cls.__doc__) return cmd.replace('/', '_'), fnc def Interact(self): import mailpile.util mailpile.util.QUITTING = False self._session.interactive = self._session.ui.interactive = True try: self._session.config.prepare_workers(self._session, daemons=True) mailpile.app.Interact(self._session) except KeyboardInterrupt: pass finally: mailpile.util.QUITTING = mailpile.util.QUITTING or True self._session.config.stop_workers() self._session.interactive = self._session.ui.interactive = False ================================================ FILE: mailpile/__main__.py ================================================ import sys from mailpile.app import Main def main(): Main(sys.argv[1:]) if __name__ == "__main__": main() ================================================ FILE: mailpile/app.py ================================================ from __future__ import print_function import getopt import gettext import locale import os import re import sys import traceback import mailpile.util import mailpile.config.defaults import mailpile.platforms from mailpile.commands import COMMANDS, Command, Action from mailpile.config.manager import ConfigManager from mailpile.conn_brokers import DisableUnbrokeredConnections from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.plugins import PluginManager from mailpile.plugins.core import Help, HelpSplash, HealthCheck from mailpile.plugins.core import Load, Rescan, Quit from mailpile.plugins.motd import MessageOfTheDay from mailpile.plugins.setup_magic import Setup from mailpile.ui import ANSIColors, Session, UserInteraction, Completer from mailpile.util import * _plugins = PluginManager(builtin=__file__) # This makes sure mailbox "plugins" get loaded... has to go somewhere? from mailpile.mailboxes import * # This is also a bit silly, should be somewhere else? Help.ABOUT = mailpile.config.defaults.ABOUT # We may try to load readline later on... maybe? readline = None ##[ Main ]#################################################################### def threaded_raw_input(prompt): """These shenigans are necessary to let Quit work reliably.""" def reader(container): try: line = raw_input(prompt).decode('utf-8').strip() container.append(line) except EOFError: pass o = [] t = threading.Thread(target=reader, args=(o,)) t.daemon = True t.start() while t.isAlive() and not mailpile.util.QUITTING: t.join(timeout=1) if not o: raise EOFError() return o[0] def write_readline_history(session): try: if session.config.sys.history_length > 0: readline.write_history_file(session.config.history_file()) else: safe_remove(session.config.history_file()) except (OSError, AttributeError, IOError): pass def CatchUnixSignals(session): def quit_app(sig, stack): Quit(session, 'quit').run() def reload_app(sig, stack): pass try: import signal if os.name != 'nt': signal.signal(signal.SIGTERM, quit_app) signal.signal(signal.SIGQUIT, quit_app) signal.signal(signal.SIGUSR1, reload_app) else: signal.signal(signal.SIGTERM, quit_app) except ImportError: pass def FriendlyPipeTransform(session, opt): """Shell-like syntax to invoke the pipe command or change render modes.""" old_render_mode = None if session.config.prefs.friendly_pipes: if ' |' in opt: cmd, pipe = opt.split(' |', 1) opt = 'pipe %s -- %s' % (pipe.strip(), cmd.strip()) elif re.search(' >\S+$', opt): cmd, pipe = opt.rsplit(' >', 1) opt = 'pipe >%s -- %s' % (pipe, cmd.strip()) opt = opt.replace('\\|', '|').replace('\\>', '>') if re.search(' :(json|j?html|text|\S+.html(?:!\S+))$', opt): old_render_mode = session.ui.render_mode opt, session.ui.render_mode = opt.rsplit(' :', 1) return old_render_mode, opt def Interact(session): global readline try: import readline as rl # Unix-only readline = rl except ImportError: pass try: if readline: readline.read_history_file(session.config.history_file()) readline.set_completer_delims(Completer.DELIMS) readline.set_completer(Completer(session).get_completer()) for opt in ["tab: complete", "set show-all-if-ambiguous on"]: readline.parse_and_bind(opt) except IOError: pass # Negative history means no saving state to disk. history_length = session.config.sys.history_length if readline is None: pass # history currently not supported under Windows / Mac elif history_length >= 0: readline.set_history_length(history_length) else: readline.set_history_length(-history_length) try: prompt = session.ui.term.color('mailpile> ', color=session.ui.term.BLACK, weight=session.ui.term.BOLD, readline=(readline is not None)) while not mailpile.util.QUITTING: try: with session.ui.term: if Setup.Next(session.config, 'anything') != 'anything': session.ui.notify( _('Mailpile is unconfigured, please run `setup`' ' or visit the web UI.')) session.ui.block() opt = threaded_raw_input(prompt) except KeyboardInterrupt: session.ui.unblock(force=True) session.ui.notify(_('Interrupted. ' 'Press CTRL-D or type `quit` to quit.')) continue session.ui.term.check_max_width() session.ui.unblock(force=True) if opt: old_render_mode, opt = FriendlyPipeTransform(session, opt) if ' ' in opt: opt, arg = opt.split(' ', 1) else: arg = '' try: result = Action(session, opt, arg) session.ui.block() session.ui.display_result(result) except UsageError as e: session.fatal_error(unicode(e)) except UrlRedirectException as e: session.fatal_error('Tried to redirect to: %s' % e.url) if old_render_mode is not None: session.ui.render_mode = old_render_mode except EOFError: print() finally: session.ui.unblock(force=True) write_readline_history(session) class InteractCommand(Command): SYNOPSIS = (None, 'interact', None, None) ORDER = ('Internals', 2) CONFIG_REQUIRED = False RAISES = (KeyboardInterrupt,) def command(self): session, config = self.session, self.session.config session.interactive = True if mailpile.platforms.TerminalSupportsAnsiColors(): session.ui.term = ANSIColors() # Ensure we have a working GnuPG self._gnupg().common_args(will_send_passphrase=True) # Create and start the rest of the threads, load the index. if config.loaded_config: Load(session, '').run(quiet=True) else: config.prepare_workers(session, daemons=True) # Note: We do *not* update the MOTD on startup, to keep things # fast, and to avoid leaking our IP on setup, before Tor # has been configured. splash = HelpSplash(session, 'help', []).run() motd = MessageOfTheDay(session, 'motd', ['--noupdate']).run() session.ui.display_result(splash) print() # FIXME: This is a hack! session.ui.display_result(motd) Interact(session) return self._success(_('Ran interactive shell')) class WaitCommand(Command): SYNOPSIS = (None, 'wait', None, None) ORDER = ('Internals', 2) CONFIG_REQUIRED = False RAISES = (KeyboardInterrupt,) def command(self): self.session.ui.display_result(HelpSplash(self.session, 'help', [] ).run(interactive=False)) while not mailpile.util.QUITTING: time.sleep(1) return self._success(_('Did nothing much for a while')) def Main(args): try: mailpile.platforms.DetectBinaries(_raise=OSError) except OSError as e: binary = str(e).split()[0] sys.stderr.write(""" Required binary missing or unusable: %s If you know where it is, or would like to skip this test and run Mailpile anyway, you can set one of the following environment variables: MAILPILE_%s="/path/to/binary" or MAILPILE_IGNORE_BINARIES="%s" # Can be a space-separated list Note that skipping a binary check may cause the app to become unstable or fail in unexpected ways. If it breaks you get to keep both pieces! """ % (e, binary.upper(), binary)) sys.exit(1) # Enable our connection broker, try to prevent badly behaved plugins from # bypassing it. DisableUnbrokeredConnections() # Bootstrap translations until we've loaded everything else mailpile.i18n.ActivateTranslation(None, ConfigManager, None) try: # Create our global config manager and the default (CLI) session config = ConfigManager(rules=mailpile.config.defaults.CONFIG_RULES) session = Session(config) cli_ui = session.ui = UserInteraction(config) session.main = True try: CatchUnixSignals(session) config.clean_tempfile_dir() config.load(session) except IOError: if config.sys.debug: session.ui.error(_('Failed to decrypt configuration, ' 'please log in!')) HealthCheck(session, None, []).run() config.prepare_workers(session) except AccessError as e: session.ui.error('Access denied: %s\n' % e) sys.exit(1) try: try: if '--login' in args: a1 = args[:args.index('--login') + 1] a2 = args[len(a1):] else: a1, a2 = args, [] allopts = [] for argset in (a1, a2): shorta, longa = '', [] for cls in COMMANDS: shortn, longn, urlpath, arglist = cls.SYNOPSIS[:4] if arglist: if shortn: shortn += ':' if longn: longn += '=' if shortn: shorta += shortn if longn: longa.append(longn.replace(' ', '_')) opts, args = getopt.getopt(argset, shorta, longa) allopts.extend(opts) for opt, arg in opts: session.ui.display_result(Action( session, opt.replace('-', ''), arg.decode('utf-8'))) if args: session.ui.display_result(Action( session, args[0], ' '.join(args[1:]).decode('utf-8'))) except (getopt.GetoptError, UsageError) as e: session.fatal_error(unicode(e)) if (not allopts) and (not a1) and (not a2): InteractCommand(session).run() except KeyboardInterrupt: pass except: traceback.print_exc() finally: write_readline_history(session) # Make everything in the background quit ASAP... mailpile.util.LAST_USER_ACTIVITY = 0 mailpile.util.QUITTING = mailpile.util.QUITTING or True if config.plugins: config.plugins.process_shutdown_hooks() config.stop_workers() if config.index: config.index.save_changes() if config.event_log: config.event_log.close() session.ui.display_result(Action(session, 'cleanup', '')) if session.interactive and config.sys.debug: session.ui.display_result(Action(session, 'ps', '')) # Remove anything that we couldn't remove before safe_remove() # Restart the app if that's what was requested if mailpile.util.QUITTING == 'restart': os.execv(sys.argv[0], sys.argv) _plugins.register_commands(InteractCommand, WaitCommand) if __name__ == "__main__": Main(sys.argv[1:]) ================================================ FILE: mailpile/auth.py ================================================ import time from urlparse import parse_qs, urlparse from urllib import quote, urlencode from mailpile.commands import Command from mailpile.crypto.gpgi import GnuPG from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.plugins import PluginManager from mailpile.security import SecurePassphraseStorage from mailpile.util import * GLOBAL_LOGIN_LOCK = CryptoLock() class UserSession(object): EXPIRE_AFTER = 7 * 24 * 3600 def __init__(self, ts=None, auth=None, data=None): self.ts = ts or time.time() self.auth = auth self.data = data or {} def is_expired(self, now=None): return (self.ts < (now or time.time()) - self.EXPIRE_AFTER) def update_ts(self): self.ts = time.time() class UserSessionCache(dict): def delete_expired(self, now=None): now = now or time.time() for k in self.keys(): if self[k].is_expired(now=now): del self[k] def VerifyAndStorePassphrase(config, passphrase=None, sps=None, key=None): if passphrase and not sps: sps = SecurePassphraseStorage(passphrase) passphrase = 'this probably does not really overwrite :-( ' safe_assert( (sps is not None) and (config.load_master_key(sps))) # Fun side effect: changing the passphrase invalidates the message cache import mailpile.mailutils.emails mailpile.mailutils.emails.ClearParseCache(full=True) return sps def SetLoggedIn(cmd, user=None, redirect=False, session_id=None): user = user or 'DEFAULT' sid = session_id or cmd.session.ui.html_variables.get('http_session') if sid: if cmd: cmd.session.ui.debug('Logged in %s as %s' % (sid, user)) SESSION_CACHE[sid] = UserSession(auth=user, data={ 't': '%x' % int(time.time()), }) if cmd: if redirect: return cmd._do_redirect() else: return True def CheckPassword(config, username, password): # FIXME: Do something with the username username = username or 'DEFAULT' sps = config.passphrases and config.passphrases.get(username) return sps.compare(password) and username def IndirectPassword(config, pwd): pp = pwd.split(':') if len(pp) > 2 and pp[0] == '_SECRET_': if pp[1] in config.secrets: return config.secrets[pp[1]].password if pp[1] in config.passphrases: return config.passphrases[pp[1]].get_passphrase() return pwd SESSION_CACHE = UserSessionCache() LOGIN_FAILURES = [] def LogoutAll(): for k in list(SESSION_CACHE.keys()): del SESSION_CACHE[k] class Authenticate(Command): """Authenticate a user (log in)""" SYNOPSIS = (None, 'login', 'auth/login', None) ORDER = ('Internals', 5) SPLIT_ARG = False IS_INTERACTIVE = True CONFIG_REQUIRED = False HTTP_AUTH_REQUIRED = False HTTP_STRICT_VARS = False HTTP_CALLABLE = ('GET', 'POST') HTTP_POST_VARS = { 'user': 'User to authenticate as', 'pass': 'Password or passphrase' } @classmethod def RedirectBack(cls, url, data): qs = [(k, v.encode('utf-8')) for k, vl in data.iteritems() for v in vl if k not in ['_method', '_path'] + cls.HTTP_POST_VARS.keys()] qs = urlencode(qs) url = ''.join([url, '?%s' % qs if qs else '']) raise UrlRedirectException(url) def _result(self, result=None): global LOGIN_FAILURES result = result or {} result['login_banner'] = self.session.config.sys.login_banner result['login_failures'] = LOGIN_FAILURES return result def _error(self, message, info=None, result=None): global LOGIN_FAILURES LOGIN_FAILURES.append(int(time.time())) time.sleep(min(5, 0.1 + len(LOGIN_FAILURES) / 2)) return Command._error(self, message, info=info, result=self._result(result)) def _success(self, message, result=None): return Command._success(self, message, result=self._result(result)) def _do_redirect(self): path = self.data.get('_path', [None])[0] # These are here to prevent people from abusing this to redirect to # arbitrary URLs on the Internet. if path: url = urlparse(path) safe_assert(not url.scheme and not url.netloc) if (path and not path[1:].startswith(DeAuthenticate.SYNOPSIS[2] or '!') and not path[1:].startswith(self.SYNOPSIS[2] or '!')): self.RedirectBack(self.session.config.sys.http_path + path, self.data) else: raise UrlRedirectException('%s/' % self.session.config.sys.http_path) def _do_login(self, user, password, load_index=False, redirect=False): global LOGIN_FAILURES session, config = self.session, self.session.config session_id = self.session.ui.html_variables.get('http_session') # This prevents folks from sending us a DEFAULT user (upper case), # which is an internal security bypass below. user = user and user.lower() if not user: try: # Verify the passphrase if CheckPassword(config, None, password): sps = config.passphrases['DEFAULT'] else: sps = VerifyAndStorePassphrase(config, passphrase=password) if sps: # Load the config and index, if necessary config = self._config() self._idx(wait=False) if load_index: try: while not config.index: time.sleep(1) except KeyboardInterrupt: pass session.ui.debug('Good passphrase for %s' % session_id) self.record_user_activity() LOGIN_FAILURES = [] return self._success(_('Hello world, welcome!'), result={ 'authenticated': SetLoggedIn(self, redirect=redirect) }) else: session.ui.debug('No GnuPG, checking DEFAULT user') # No GnuPG, see if there is a DEFAULT user in the config user = 'DEFAULT' except (AssertionError, IOError): session.ui.debug('Bad passphrase for %s' % session_id) return self._error(_('Invalid password, please try again')) if user in config.logins or user == 'DEFAULT': # FIXME: Salt and hash the password, check if it matches # the entry in our user/password list (TODO). # NOTE: This hack effectively disables auth without GnUPG if user == 'DEFAULT': session.ui.debug('FIXME: Unauthorized login allowed') return self._logged_in(redirect=redirect) raise Exception('FIXME') return self._error(_('Incorrect username or password')) def command(self): session_id = self.session.ui.html_variables.get('http_session') if self.data.get('_method', '') == 'POST': if 'pass' in self.data: with GLOBAL_LOGIN_LOCK: return self._do_login(self.data.get('user', [None])[0], self.data['pass'][0], redirect=True) elif not self.data: password = self.session.ui.get_password(_('Your password: ')) return self._do_login(None, password, load_index=True) elif (session_id in SESSION_CACHE and SESSION_CACHE[session_id].auth and '_method' in self.data): self._do_redirect() return self._success(_('Please log in')) class DeAuthenticate(Command): """De-authenticate a user (log out)""" SYNOPSIS = (None, 'logout', 'auth/logout', '[]') ORDER = ('Internals', 5) SPLIT_ARG = False IS_INTERACTIVE = True CONFIG_REQUIRED = False HTTP_AUTH_REQUIRED = False HTTP_CALLABLE = ('GET', 'POST') def command(self): # FIXME: Should this only be a POST request? # FIXME: This needs CSRF protection. session_id = self.session.ui.html_variables.get('http_session') if self.args and not session_id: session_id = self.args[0] if session_id: try: self.session.ui.debug('Logging out %s' % session_id) del SESSION_CACHE[session_id] return self._success(_('Goodbye!')) except KeyError: pass return self._error(_('No session found!')) class SetPassphrase(Command): """Manage storage of passwords (passphrases)""" SYNOPSIS = (None, 'set/password', 'settings/set/password', ' [store|cache-only[:]|fail|forget]') ORDER = ('Config', 9) SPLIT_ARG = True IS_INTERACTIVE = True IS_USER_ACTIVITY = True CONFIG_REQUIRED = True HTTP_AUTH_REQUIRED = True HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = { 'id': 'KeyID or account name', 'mailsource': 'Mail source ID', 'mailroute': 'Mail route ID', 'is_locked': 'Assume key is locked'} HTTP_POST_VARS = { 'password': 'KeyID or account name', 'policy-ttl': 'Combined policy and TTL', 'policy': 'store|cache-only|fail|forget', 'ttl': 'Seconds after which it expires, -1 = never', 'update_mailsources': 'If true, update mail source settings', 'update_mailroutes': 'If true, update mail route settings', 'redirect': 'URL to redirect to on success'} def _get_profiles(self): return self.session.config.vcards.find_vcards([], kinds=['profile']) def _get_policy(self, fingerprint): if fingerprint in self.session.config.secrets: return self.session.config.secrets[fingerprint].policy elif fingerprint in self.session.config.passphrases: return 'cache-only' return None def _massage_key_info(self, fingerprint, key_info, profiles=None, is_locked=False): config = self.session.config fingerprint = fingerprint.lower() key_info["uids"].sort( key=lambda k: (k.get("name"), k.get("email"), k.get("comment"))) if not is_locked: key_info['policy'] = self._get_policy(fingerprint) if key_info['policy'] is None: del key_info['policy'] key_info["accounts"] = [] if profiles is None: profiles = self._get_profiles() for vc in profiles: vc_pgp_key = (vc.pgp_key or '').lower() if vc_pgp_key == fingerprint: key_info["accounts"].append({ 'name': vc.fn, 'email': vc.email, 'rid': vc.random_uid}) return key_info def _lookup_key(self, keyid, **kwargs): keylist = self._gnupg().list_secret_keys(selectors=[keyid]) if len(keylist) != 1: raise ValueError("Too many or too few keys found!") fingerprint, key_info = keylist.keys()[0], keylist.values()[0] return fingerprint, self._massage_key_info( fingerprint, key_info, **kwargs) def _list_keys(self, **kwargs): keylist = self._gnupg().list_secret_keys() profiles = self._get_profiles() for fingerprint, key_info in keylist.iteritems(): self._massage_key_info(fingerprint, key_info, profiles=profiles, **kwargs) return keylist def _get_account(self, cfg): username = cfg.username if cfg.username and '@' not in cfg.username: username = '%s@%s' % (username, cfg.host) return username def _user_fingerprint(self, username): return username.replace('@', '_').replace('.', '_').lower() def _list_accounts(self, only=None): accounts = {} def _add_account(cfg, which): username = self._get_account(cfg) if (username and ((username == only) or only is None) and cfg.auth_type == 'password'): if username in accounts: accounts[username][which] = cfg.host else: fingerprint = self._user_fingerprint(username) accounts[username] = { which: cfg.host, 'username': username, 'policy': self._get_policy(fingerprint)} if accounts[username]['policy'] is None: del accounts[username]['policy'] for msid, route in self.session.config.routes.iteritems(): _add_account(route, 'route') for msid, source in self.session.config.sources.iteritems(): _add_account(source, 'source') return accounts def _account_details(self, account): return self._list_accounts(only=account).get(account) def _check_master_password(self, password, account=None, fingerprint=None): return CheckPassword(self.session.config, None, password) def _check_password(self, password, account=None, fingerprint=None): if account: # We're going to keep punting on this for a while... return True elif fingerprint: sps = SecurePassphraseStorage(password) gpg = GnuPG(self.session.config) status, sig = gpg.sign('OK', fromkey=fingerprint, passphrase=sps) return (status == 0) else: return True def _prepare_result(self, account=None, keyid=None, is_locked=False): if account: fingerprint = self._user_fingerprint(account) result = {'account': self._account_details(account)} elif keyid: fingerprint, info = self._lookup_key(keyid, is_locked=is_locked) result = {'key': info} else: fingerprint = None result = { 'keylist': self._list_keys(is_locked=is_locked), 'accounts': self._list_accounts()} return fingerprint, result def command(self): config = self.session.config policyttl = self.args[1] if (len(self.args) > 1) else 'cache-only:-1' is_locked = self.data.get('is_locked', [0])[0] if 'policy-ttl' in self.data: policyttl = self.data['policy-ttl'][0] if ':' in policyttl: policy, ttl = policyttl.split(':') else: policy, ttl = policyttl, -1 if 'policy' in self.data: policy = self.data['policy'][0] if 'ttl' in self.data: ttl = self.data['policy'][0] ttl = float(ttl) fingerprint = info = account = keyid = None which = self.args[0] if self.args else self.data.get('id', [None])[0] if which and '@' in which: account = which else: keyid = which if not account and not keyid: msid = self.data.get('mailsource', [None])[0] if msid: account = self._get_account(config.sources[msid]) mrid = self.data.get('mailroute', [None])[0] if mrid: account = self._get_account(config.routes[mrid]) fingerprint, result = self._prepare_result( account=account, keyid=keyid, is_locked=is_locked) if policy in ('display', 'unprotect'): pass_prompt = _('Enter your Mailpile password') pass_check = self._check_master_password else: pass_prompt = _('Enter your password') pass_check = self._check_password if self.data.get('_method', None) == 'GET': return self._success(pass_prompt, result) safe_assert(fingerprint is not None) fingerprint = fingerprint.lower() if fingerprint in config.secrets: if config.secrets[fingerprint].policy == 'protect': if policy not in ('unprotect', 'display'): result['error'] = _('That key is managed by Mailpile,' ' it cannot be changed directly.') return self._error(_('Protected secret'), result=result) if self.data.get('_method', None) == 'POST': password = self.data.get('password', [None])[0] update_ms = self.data.get('update_mailsources', [False])[0] update_mr = self.data.get('update_mailroutes', [False])[0] else: password = self.session.ui.get_password(pass_prompt + ': ') update_ms = update_mr = (account is not None) if update_ms or update_mr: safe_assert(account is not None) if not pass_check(password, account=account, fingerprint=fingerprint): result['error'] = _('Password incorrect! Try again?') return self._error(_('Incorrect password'), result=result) def _account_matches(cfg): return (account == cfg.username or account == '%s@%s' % (cfg.username, cfg.host)) def happy(msg, refresh=True, changed=True): if changed: # Fun side effect: changing the passphrase invalidates the # message cache import mailpile.mailutils.emails mailpile.mailutils.emails.ClearParseCache(full=True) indirect_pwd = '_SECRET_:%s:%s' % (fingerprint, time.time()) if update_ms: for msid, source in config.sources.iteritems(): if _account_matches(source): source.password = indirect_pwd if update_mr: for msid, route in config.routes.iteritems(): if _account_matches(route): route.password = indirect_pwd self._background_save(config=True) redirect = self.data.get('redirect', [None])[0] if redirect: raise UrlRedirectException(redirect) result['op_completed'] = policy if refresh: fp, r = self._prepare_result(account=account, keyid=keyid) result.update(r) return self._success(msg, result) if policy == 'display': if fingerprint in config.passphrases: pwd = config.passphrases[fingerprint].get_passphrase() elif fingerprint in config.secrets: pwd = config.secrets[fingerprint].password else: return self._error(_('No password found'), result=result) result['stored_password'] = pwd return happy(_('Retrieved stored password'), refresh=False, changed=False) if policy == 'forget': if fingerprint in config.passphrases: del config.passphrases[fingerprint] if fingerprint in config.secrets: config.secrets[fingerprint] = {'policy': 'fail'} del config.secrets[fingerprint] return happy(_('Password forgotten!')) if policy == 'fail': if fingerprint in config.passphrases: del config.passphrases[fingerprint] config.secrets[fingerprint] = {'policy': policy} return happy(_('Password will never be stored')) if policy == 'store': if fingerprint in config.passphrases: del config.passphrases[fingerprint] config.secrets[fingerprint] = { 'password': password, 'policy': policy} return happy(_('Password remembered!')) elif policy == 'cache-only' and password: sps = SecurePassphraseStorage(password) if ttl > 0: sps.expiration = time.time() + ttl config.passphrases[fingerprint] = sps if fingerprint in config.secrets: config.secrets[fingerprint] = {'policy': 'fail'} del config.secrets[fingerprint] return happy(_('The password has been stored temporarily')) else: return self._error(_('Invalid password policy'), result=result) plugin_manager = PluginManager(builtin=True) plugin_manager.register_commands(Authenticate, DeAuthenticate, SetPassphrase) ================================================ FILE: mailpile/command_cache.py ================================================ import time import mailpile.util from mailpile.commands import Command from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.plugins import PluginManager from mailpile.util import * from mailpile.ui import Session, BackgroundInteraction _plugins = PluginManager(builtin=__name__) class CommandCache(object): # # This is an in-memory cache of commands and results we may want to # refresh in the background and/or reuse. # # The way this works: # - Cache-able commands generate a fingerprint describing themselves. # - If the fingerprint is found in cache, reuse the result object. # - Otherwise, run, generate a list of requirements and cache all of: # the command object itself, the requirements, the result object. # - Internal state changes (tag ops, new mail, etc.) call mark_dirty() # describing which assets (requirements) have changed. # - Periodically, the cache is refreshed, which re-runs any dirtied # commands and fires events notifying the UI about changes. # # Examples of requirements: # # - Search terms, eg. 'in:inbox' or 'potato' or 'salad' # - Messages: 'msg:INDEX' where INDEX is a number (not a MID) # - Threads: 'thread:MID' were MID is the thread ID. # - The app configuration: '!config' # def __init__(self, debug=None): self.debug = debug or (lambda s: None) self.lock = UiRLock() self._lag = 0.1 self.cache = {} # id -> [exp, req, ss, cmd_obj, res_obj, added] self.dirty = [] # (ts, req): Requirements that changed & when self._dirty_ttl = 10 def cache_result(self, fprint, expires, req, cmd_obj, result_obj): with self.lock: # Make a snapshot of the session, as it provides context ui = BackgroundInteraction(cmd_obj.session.config, log_parent=cmd_obj.session.ui) ss = Session.Snapshot(cmd_obj.session, ui=ui) # Note: We cache this even if the requirements are "dirty", # as mere presence in the cache makes this a candidate # for refreshing. self.cache[str(fprint)] = [expires, req, ss, cmd_obj, result_obj, time.time()] self.debug('Cached %s, req=%s' % (fprint, sorted(list(req)))) def get_result(self, fprint, dirty_check=True, extend=300): with self.lock: exp, req, ss, co, result_obj, a = match = self.cache[fprint] if dirty_check: recent = (a > time.time() - self._lag) dirty = (req & self.dirty_set(after=a)) if recent or dirty: # If item is too new, or requirements are dirty, pretend this # item does not exist. self.debug('Suppressing cache result %s, recent=%s dirty=%s' % (fprint, recent, sorted(list(dirty)))) raise KeyError(fprint) match[0] = time.time() + extend co.session = result_obj.session = ss self.debug('Returning cached result for %s' % fprint) return result_obj def dirty_set(self, after=0): dirty = set(['!timedout']) with self.lock: for ts, req in self.dirty: if (after == 0) or (ts > after): dirty |= req return dirty def mark_dirty(self, requirements): with self.lock: self.dirty.append((time.time(), set(requirements))) self.debug('Marked dirty: %s' % sorted(list(requirements))) def refresh(self, extend=0, runtime=5, event_log=None): if mailpile.util.LIVE_USER_ACTIVITIES > 0: self.debug('Skipping cache refresh, user is waiting.') return started = now = time.time() with self.lock: # Expire things from the cache expired = set([f for f in self.cache if self.cache[f][0] < now]) for fp in expired: del self.cache[fp] # Expire things from the dirty set self.dirty = [(ts, req) for ts, req in self.dirty if ts >= (now - self._dirty_ttl)] # Decide which fingerprints to look at this time around fingerprints = list(self.cache.keys()) refreshed = [] fingerprints.sort(key=lambda k: -self.cache[k][0]) for fprint in fingerprints: req = None try: e, req, ss, co, ro, a = self.cache[fprint] now = time.time() dirty = (req & self.dirty_set(after=a)) if (a + self._lag < now) and dirty: if now < started + runtime: play_nice_with_threads() co.session = ro.session = ss ro = co.refresh() if extend > 0: e = min(e + extend, now + 5*extend) if '!timedout' in req: req.remove('!timedout') with self.lock: # Make sure we do not overwrite new results from # elsewhere at this time. if self.cache[fprint][-1] == a: e = max(e, self.cache[fprint][0]) # Clobber? self.cache[fprint] = [e, req, ss, co, ro, now] refreshed.append(fprint) else: # Out of time, mark as dirty. req.add('!timedout') except (ValueError, IndexError, TypeError): # Treat broken things as if they had timed out if req: req.add('!timedout') if refreshed and event_log: event_log.log(message=_('New results are available'), source=self, data={'cache_ids': refreshed}, flags=Event.COMPLETE) self.debug('Refreshed: %s' % refreshed) class Cached(Command): """Fetch results from the command cache.""" SYNOPSIS = (None, 'cached', 'cached', '[]') ORDER = ('Internals', 7) HTTP_QUERY_VARS = {'id': 'Cache ID of command to redisplay'} IS_USER_ACTIVITY = False LOG_NOTHING = True def max_age(self): # Allow result to be cached by the browser for 2 seconds; we do # this to facilitate cross-tab sharing of cache results. return 2 # Warning: This depends on internals of Command, how things are run there. def run(self): try: cid = self.args[0] if self.args else self.data.get('id', [None])[0] rv = self.session.config.command_cache.get_result(cid) self.session.copy(rv.session) rv.session.ui.render_mode = self.session.ui.render_mode return rv except: self._starting() self._error(self.FAILURE % {'name': self.name, 'args': ' '.join(self.args)}) return self._finishing(False) _plugins.register_commands(Cached) ================================================ FILE: mailpile/commands.py ================================================ # The basic Mailpile command framework. # # TODO: Merge with plugins/ the division is obsolete and artificial. # import json import os import re import shlex import traceback import time import mailpile.util import mailpile.security as security from mailpile.crypto.gpgi import GnuPG from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.util import * from mailpile.vfs import vfs # Commands starting with _ don't get single-letter shortcodes... COMMANDS = [] COMMAND_GROUPS = ['Internals', 'Config', 'Searching', 'Tagging', 'Composing'] class Command(object): """Generic command object all others inherit from""" SYNOPSIS = (None, # CLI shortcode, e.g. A: None, # CLI shortname, e.g. add None, # API endpoint, e.g. sys/addmailbox None) # Positional argument list SYNOPSIS_ARGS = None # New-style positional argument list API_VERSION = None UI_CONTEXT = None IS_USER_ACTIVITY = False IS_HANGING_ACTIVITY = False IS_INTERACTIVE = False CONFIG_REQUIRED = True COMMAND_CACHE_TTL = 0 # < 1 = Not cached CHANGES_SESSION_CONTEXT = False FAILURE = 'Failed: %(name)s %(args)s' ORDER = (None, 0) SPLIT_ARG = True # Uses shlex by default RAISES = (UsageError, UrlRedirectException) WITH_CONTEXT = () COMMAND_SECURITY = None # Event logging settings LOG_NOTHING = False LOG_ARGUMENTS = True LOG_PROGRESS = False LOG_STARTING = '%(name)s: Starting' LOG_FINISHED = '%(name)s: %(message)s' # HTTP settings (note: security!) HTTP_CALLABLE = ('GET', ) HTTP_POST_VARS = {} HTTP_QUERY_VARS = {} HTTP_BANNED_VARS = {} HTTP_STRICT_VARS = True HTTP_AUTH_REQUIRED = True class CommandResult: def __init__(self, command_obj, session, command_name, doc, result, status, message, template_id=None, kwargs={}, error_info={}): self.session = session self.command_obj = command_obj self.command_name = command_name self.kwargs = {} self.kwargs.update(kwargs) self.template_id = template_id self.doc = doc self.result = result self.status = status self.error_info = {} self.error_info.update(error_info) self.message = message self.rendered = {} self.renderers = { 'json': self.as_json, 'html': self.as_html, 'text': self.as_text, 'css': self.as_css, 'csv': self.as_csv, 'rss': self.as_rss, 'xml': self.as_xml, 'txt': self.as_txt, 'js': self.as_js } def __nonzero__(self): return (self.result and True or False) def as_(self, what, *args, **kwargs): if args or kwargs: # Args render things un-cacheable. return self.renderers.get(what)(*args, **kwargs) if what not in self.rendered: self.rendered[what] = self.renderers.get(what, self.as_text)() return self.rendered[what] def as_text(self): if isinstance(self.result, bool): happy = '%s: %s' % (self.result and _('OK') or _('Failed'), self.message or self.doc) if not self.result and self.error_info: return '%s\n%s' % (happy, json.dumps(self.error_info, indent=4, default=mailpile.util.json_helper)) else: return happy elif isinstance(self.result, (dict, list, tuple)): return json.dumps(self.result, indent=4, sort_keys=True, default=mailpile.util.json_helper) else: return unicode(self.result) __str__ = lambda self: self.as_text() __unicode__ = lambda self: self.as_text() def as_dict(self): from mailpile.urlmap import UrlMap um = UrlMap(self.session) rv = { 'command': self.command_name, 'state': { 'command_url': um.ui_url(self.command_obj), 'context_url': um.context_url(self.command_obj), 'query_args': self.command_obj.state_as_query_args(), 'cache_id': self.command_obj.cache_id(), 'context': self.command_obj.context or '' }, 'status': self.status, 'message': self.message, 'result': self.result, 'event_id': self.command_obj.event.event_id, 'elapsed': '%.3f' % self.session.ui.time_elapsed, } csrf_token = self.session.ui.html_variables.get('csrf_token') if csrf_token: rv['state']['csrf_token'] = csrf_token if self.error_info: rv['error'] = self.error_info for ui_key in [k for k in self.command_obj.data.keys() if k.startswith('ui_')]: rv[ui_key] = self.command_obj.data[ui_key][0] ev = self.command_obj.event if ev and ev.data.get('password_needed'): rv['password_needed'] = ev.private_data['password_needed'] return rv def as_csv(self, template=None, result=None): result = self.result if (result is None) else result if (isinstance(result, (list, tuple)) and (not result or isinstance(result[0], (list, tuple)))): import csv, StringIO output = StringIO.StringIO() writer = csv.writer(output, dialect='excel') for row in result: writer.writerow([unicode(r).encode('utf-8') for r in row]) return output.getvalue().decode('utf-8') else: return '' def as_json(self): return self.session.ui.render_json(self.as_dict()) def as_html(self, template=None): return self.as_template('html', template) def as_js(self, template=None): return self.as_template('js', template) def as_css(self, template=None): return self.as_template('css', template) def as_rss(self, template=None): return self.as_template('rss', template) def as_xml(self, template=None): return self.as_template('xml', template) def as_txt(self, template=None): return self.as_template('txt', template) def as_template(self, ttype, mode=None, wrap_in_json=False, template=None): cache_id = ''.join(('j' if wrap_in_json else '', ttype, '/' if template else '', template or '', ':', mode or 'full')) if cache_id in self.rendered: return self.rendered[cache_id] tpath = self.command_obj.template_path( ttype, template_id=self.template_id, template=template) data = self.as_dict() data['title'] = self.message data['render_mode'] = mode or 'full' data['render_template'] = template or 'index' rendering = self.session.ui.render_web(self.session.config, [tpath], data) if wrap_in_json: data['result'] = rendering self.rendered[cache_id] = self.session.ui.render_json(data) else: self.rendered[cache_id] = rendering return self.rendered[cache_id] def __init__(self, session, name=None, arg=None, data=None, async=False): self.session = session self.context = None self.name = self.SYNOPSIS[1] or self.SYNOPSIS[2] or name self.data = data or {} self.status = 'unknown' self.message = name self.error_info = {} self.result = None self.run_async = async if type(arg) in (type(list()), type(tuple())): self.args = tuple(arg) elif arg: if self.SPLIT_ARG is True: try: self.args = tuple([a.decode('utf-8') for a in shlex.split(arg.encode('utf-8'))]) except (ValueError, UnicodeEncodeError, UnicodeDecodeError): raise UsageError(_('Failed to parse arguments')) else: self.args = (arg, ) else: self.args = tuple([]) if 'arg' in self.data: self.args = tuple(list(self.args) + self.data['arg']) self._create_event() def state_as_query_args(self): args = {} if self.args: args['arg'] = self._sloppy_copy(self.args) args.update(self._sloppy_copy(self.data)) return args def cache_id(self, sqa=None): if self.COMMAND_CACHE_TTL < 1: return '' from mailpile.urlmap import UrlMap args = sorted(list((sqa or self.state_as_query_args()).iteritems())) args += '/%d' % self.session.ui.term.max_width # The replace() stuff makes these usable as CSS class IDs return ('%s-%s' % (UrlMap(self.session).ui_url(self), md5_hex(str(args)) )).replace('/', '-').replace('.', '-') def cache_requirements(self, result): raise NotImplementedError('Cachable commands should override this, ' 'returning a set() of requirements.') def cache_result(self, result): if self.COMMAND_CACHE_TTL > 0: try: cache_id = self.cache_id() if cache_id: self.session.config.command_cache.cache_result( cache_id, time.time() + self.COMMAND_CACHE_TTL, self.cache_requirements(result), self, result) self.session.ui.mark(_('Cached result as %s') % cache_id) except (ValueError, KeyError, TypeError, AttributeError): self._ignore_exception() def template_path(self, ttype, template_id=None, template=None): path_parts = (template_id or self.SYNOPSIS[2] or 'command').split('/') if template in (None, ttype, 'as.' + ttype): path_parts.append('index') else: # Security: The template request may come from the URL, so we # sanitize it very aggressively before heading off # to the filesystem. clean_tpl = CleanText(template.replace('.%s' % ttype, ''), banned=(CleanText.FS + CleanText.WHITESPACE)) path_parts.append(clean_tpl.clean) path_parts[-1] += '.' + ttype return os.path.join(*path_parts) def _gnupg(self, **kwargs): return GnuPG(self.session.config, event=self.event, **kwargs) def _config(self): session, config = self.session, self.session.config if not config.loaded_config: config.load(session) parent = session config.prepare_workers(session, daemons=self.IS_INTERACTIVE) if self.IS_INTERACTIVE and not config.daemons_started(): config.prepare_workers(session, daemons=True) return config def _idx(self, reset=False, wait=True, wait_all=True, quiet=False): session, config = self.session, self._config() if not reset and config.index: return config.index def __do_load2(): config.vcards.load_vcards(session) if not wait_all: session.ui.report_marks(quiet=quiet) def __do_load1(): with config.interruptable_wait_for_lock(): if reset: config.index = None session.results = [] session.searched = [] session.displayed = None idx = config.get_index(session) if wait_all: __do_load2() if not wait: session.ui.report_marks(quiet=quiet) return idx if wait: rv = __do_load1() session.ui.reset_marks(quiet=quiet) else: config.save_worker.add_task(session, 'Load', __do_load1) rv = None if not wait_all: config.save_worker.add_task(session, 'Load2', __do_load2) return rv def _background_save(self, everything=False, config=False, index=False, index_full=False, wait=False, wait_callback=None): session, cfg = self.session, self.session.config aut = cfg.save_worker.add_unique_task if everything or config: aut(session, 'Save config', lambda: cfg.save(session, force=(config == '!FORCE')), first=True) if cfg.index: cfg.flush_mbox_cache(session, clear=False, wait=wait) if index_full: aut(session, 'Save index', lambda: self._idx().save(session), first=True) elif everything or index: aut(session, 'Save index changes', lambda: self._idx().save_changes(session), first=True) if wait: wait_callback = wait_callback or (lambda: True) cfg.save_worker.do(session, 'Waiting', wait_callback) def _choose_messages(self, words, allow_ephemeral=False): msg_ids = set() all_words = [] for word in words: all_words.extend(word.split(',')) for what in all_words: if what.lower() == 'these': if self.session.displayed: b = self.session.displayed['stats']['start'] - 1 c = self.session.displayed['stats']['count'] msg_ids |= set(self.session.results[b:b + c]) else: self.session.ui.warning(_('No results to choose from!')) elif what.lower() in ('all', '!all', '=!all'): if self.session.results: msg_ids |= set(self.session.results) else: self.session.ui.warning(_('No results to choose from!')) elif what.startswith('='): try: msg_id = int(what.replace('=', ''), 36) if msg_id >= 0 and msg_id < len(self._idx().INDEX): msg_ids.add(msg_id) else: self.session.ui.warning((_('No such ID: %s') ) % (what[1:], )) except ValueError: if allow_ephemeral and '-' in what: msg_ids.add(what[1:]) else: self.session.ui.warning(_('What message is %s?' ) % (what, )) elif '-' in what: try: b, e = what.split('-') msg_ids |= set(self.session.results[int(b) - 1:int(e)]) except (ValueError, KeyError, IndexError, TypeError): self.session.ui.warning(_('What message is %s?' ) % (what, )) else: try: msg_ids.add(self.session.results[int(what) - 1]) except (ValueError, KeyError, IndexError, TypeError): self.session.ui.warning(_('What message is %s?' ) % (what, )) return msg_ids def _error(self, message, info=None, result=None): self.status = 'error' self.message = message ui_message = _('%s error: %s') % (self.name, message) if info: self.error_info.update(info) details = ' '.join(['%s=%s' % (k, info[k]) for k in info]) ui_message += ' (%s)' % details self.session.ui.mark(self.name) self.session.ui.error(ui_message) if result: return self.view(result) else: return False def _success(self, message, result=True): self.status = 'success' self.message = message ui_message = '%s: %s' % (self.name, message) self.session.ui.mark(ui_message) return self.view(result) def _read_file_or_data(self, fn): if fn in self.data: return self.data[fn] else: return vfs.open(fn, 'rb').read() def _ignore_exception(self): self.session.ui.debug(traceback.format_exc()) def _serialize(self, name, function): return function() def _background(self, name, function): session, config = self.session, self.session.config return config.scan_worker.add_task(session, name, function) def _update_event_state(self, state, log=False): self.event.flags = state self.event.data['elapsed'] = int(1000 * (time.time()-self._start_time)) if (log or self.LOG_PROGRESS) and not self.LOG_NOTHING: self.event.data['ui'] = str(self.session.ui.__class__.__name__) self.event.data['output'] = self.session.ui.render_mode if self.session.config.event_log: self.session.config.event_log.log_event(self.event) def _starting(self): self._start_time = time.time() self._update_event_state(Event.RUNNING) if self.name: self.session.ui.start_command(self.name, self.args, self.data) def _fmt_msg(self, message): return message % {'name': self.name, 'status': self.status or '', 'message': self.message or ''} def _sloppy_copy(self, data, name=None): if name and (name[:4] in ('pass', 'csrf') or 'password' in name or 'passphrase' in name): data = '(SUPPRESSED)' def copy_value(v): try: unicode(v).encode('utf-8') return unicode(v)[:1024] except (UnicodeEncodeError, UnicodeDecodeError): return '(BINARY DATA)' if isinstance(data, (list, tuple)): return [self._sloppy_copy(i, name=name) for i in data] elif isinstance(data, dict): return dict((k, self._sloppy_copy(v, name=k)) for k, v in data.iteritems()) else: return copy_value(data) def _create_event(self): private_data = {} if self.LOG_ARGUMENTS: if self.data: private_data['data'] = self._sloppy_copy(self.data) if self.args: private_data['args'] = self._sloppy_copy(self.args) self.event = self._make_command_event(private_data) def _make_command_event(self, private_data): return Event(source=self, message=self._fmt_msg(self.LOG_STARTING), flags=Event.INCOMPLETE, data={}, private_data=private_data) def _finishing(self, rv, just_cleanup=False): if just_cleanup: self._update_finished_event() return rv if not self.context: self.context = self.session.get_context( update=self.CHANGES_SESSION_CONTEXT) self.session.ui.mark(_('Generating result')) result = self.CommandResult(self, self.session, self.name, self.__doc__, rv, self.status, self.message, error_info=self.error_info) self.cache_result(result) if not self.run_async: self._update_finished_event() self.session.last_event_id = self.event.event_id return result def _update_finished_event(self): # Update the event! if self.message: self.event.message = self.message if self.error_info: self.event.private_data['error_info'] = self.error_info self.event.message = self._fmt_msg(self.LOG_FINISHED) self._update_event_state(Event.COMPLETE, log=True) self.session.ui.mark(self.event.message) self.session.ui.report_marks( details=('timing' in self.session.config.sys.debug)) if self.name: self.session.ui.finish_command(self.name) def _run_sync(self, enable_cache, *args, **kwargs): try: thread_context_push(command=self, event=self.event, session=self.session) self._starting() self._run_args = args self._run_kwargs = kwargs if (self.COMMAND_CACHE_TTL > 0 and 'http' not in self.session.config.sys.debug and enable_cache): cid = self.cache_id() try: rv = self.session.config.command_cache.get_result(cid) rv.session.ui = self.session.ui if self.CHANGES_SESSION_CONTEXT: self.session.copy(rv.session) self.session.ui.mark(_('Using pre-cached result object %s') % cid) self._finishing(True, just_cleanup=True) return rv except: pass def command(self, *args, **kwargs): if self.CONFIG_REQUIRED: if not self.session.config.loaded_config: return self._error(_('Please log in')) if mailpile.util.QUITTING: return self._error(_('Shutting down')) return self.command(*args, **kwargs) return self._finishing(command(self, *args, **kwargs)) except self.RAISES: self.status = 'success' self._finishing(True, just_cleanup=True) raise except: self._ignore_exception() self._error(self.FAILURE % {'name': self.name, 'args': ' '.join(self.args)}) return self._finishing(False) finally: thread_context_pop() def _run(self, *args, **kwargs): if self.run_async: def streetcar(): try: with MultiContext(self.WITH_CONTEXT): rv = self._run_sync(True, *args, **kwargs).as_dict() self.event.private_data.update(rv) self._update_finished_event() except: traceback.print_exc() self._starting() self._update_event_state(self.event.RUNNING, log=True) result = Command.CommandResult(self, self.session, self.name, self.__doc__, {"resultid": self.event.event_id}, "success", "Running in background") self.session.config.scan_worker.add_task(self.session, self.name, streetcar, first=True) return result else: return self._run_sync(True, *args, **kwargs) def _maybe_trigger_cache_refresh(self): if self.data.get('_method') == 'POST': def refresher(): self.session.config.command_cache.refresh( event_log=self.session.config.event_log) self.session.config.scan_worker.add_unique_task( self.session, 'post-refresh', refresher, first=True) def record_user_activity(self): mailpile.util.LAST_USER_ACTIVITY = time.time() def run(self, *args, **kwargs): if self.COMMAND_SECURITY is not None: forbidden = security.forbid_command(self) if forbidden: return self._error(forbidden) with MultiContext(self.WITH_CONTEXT): if self.IS_USER_ACTIVITY: try: self.record_user_activity() mailpile.util.LIVE_USER_ACTIVITIES += 1 rv = self._run(*args, **kwargs) self._maybe_trigger_cache_refresh() return rv finally: mailpile.util.LIVE_USER_ACTIVITIES -= 1 else: rv = self._run(*args, **kwargs) self._maybe_trigger_cache_refresh() return rv def refresh(self): self._create_event() return self._run_sync(False, *self._run_args, **self._run_kwargs) def command(self): return None def etag_data(self): return [] def max_age(self): return 0 @classmethod def view(cls, result): return result def GetCommand(name): match = [c for c in COMMANDS if name in c.SYNOPSIS[:3]] if len(match) == 1: return match[0] return None def Action(session, opt, arg, data=None): session.ui.reset_marks(quiet=True) config = session.config if not opt: return Help(session, 'help').run() # Use the COMMANDS dict by default. command = GetCommand(opt) if command: return command(session, opt, arg, data=data).run() # Tags are commands if config.loaded_config: lopt = opt.lower() found = None for tag in config.tags.values(): if lopt == tag.slug.lower(): found = tag break if not found: for tag in config.tags.values(): if lopt == tag.name.lower(): found = tag break if not found: for tag in config.tags.values(): if lopt == _(tag.name).lower(): found = tag break if found: a = 'in:%s%s%s' % (found.slug, ' ' if arg else '', arg) return GetCommand('search')(session, opt, arg=a, data=data).run() # OK, give up! raise UsageError(_('Unknown command: %s') % opt) ================================================ FILE: mailpile/config/__init__.py ================================================ ================================================ FILE: mailpile/config/base.py ================================================ from __future__ import print_function import io import json import os import ConfigParser from urllib import quote, unquote from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.util import * import mailpile.config.validators as validators class ConfigValueError(ValueError): pass def ConfigRule(*args): class _ConfigRule(list): def __init__(self): list.__init__(self, args) self._types = [] return _ConfigRule() def PublicConfigRule(*args): c = ConfigRule(*args) c._types.append('public') return c def KeyConfigRule(*args): c = ConfigRule(*args) c._types.append('key') return c # FIXME: This should be enforced somehow when variables are altered. # Run in a context? def CriticalConfigRule(*args): c = ConfigRule(*args) c._types += ['critical'] return c def ConfigPrinter(cfg, indent=''): rv = [] if isinstance(cfg, dict): pairer = cfg.iteritems() else: pairer = enumerate(cfg) for key, val in pairer: if hasattr(val, 'rules'): preamble = '[%s: %s] ' % (val._NAME, val._COMMENT) else: preamble = '' if isinstance(val, (dict, list, tuple)): if isinstance(val, dict): b, e = '{', '}' else: b, e = '[', ']' rv.append(('%s: %s%s\n%s\n%s' '' % (key, preamble, b, ConfigPrinter(val, ' '), e) ).replace('\n \n', '')) elif isinstance(val, (str, unicode)): rv.append('%s: "%s"' % (key, val)) else: rv.append('%s: %s' % (key, val)) return indent + ',\n'.join(rv).replace('\n', '\n'+indent) class InvalidKeyError(ValueError): pass class CommentedEscapedConfigParser(ConfigParser.RawConfigParser): """ This is a ConfigParser that allows embedded comments and safely escapes and encodes/decodes values that include funky characters. >>> cfg = u'[config/sys: Stuff]\\ndebug = True ; Ignored comment' >>> cecp = CommentedEscapedConfigParser() >>> cecp.readfp(io.BytesIO(cfg.encode('utf-8'))) >>> cecp.get('config/sys: Stuff', 'debug') == 'True' True >>> cecp.items('config/sys: Stuff') [(u'debug', u'True')] """ NOT_UTF8 = '%C0' # This byte is never valid at the start of an utf-8 # string, so we use it to mark binary data. SAFE = '!?: /#@<>[]()=-' def set(self, section, key, value, comment): key = unicode(key).encode('utf-8') section = unicode(section).encode('utf-8') if isinstance(value, unicode): value = quote(value.encode('utf-8'), safe=self.SAFE) elif isinstance(value, str): quoted = quote(value, safe=self.SAFE) if quoted != value: value = self.NOT_UTF8 + quoted else: value = quote(unicode(value).encode('utf-8'), safe=self.SAFE) if value.endswith(' '): value = value[:-1] + '%20' if comment: pad = ' ' * (25 - len(key) - len(value)) + ' ; ' value = '%s%s%s' % (value, pad, comment) return ConfigParser.RawConfigParser.set(self, section, key, value) def _decode_value(self, value): if value.startswith(self.NOT_UTF8): return unquote(value[len(self.NOT_UTF8):]) else: return unquote(value).decode('utf-8') def get(self, section, key): key = unicode(key).encode('utf-8') section = unicode(section).encode('utf-8') value = ConfigParser.RawConfigParser.get(self, section, key) return self._decode_value(value) def items(self, section): return [(k.decode('utf-8'), self._decode_value(i)) for k, i in ConfigParser.RawConfigParser.items(self, section)] def _MakeCheck(pcls, name, comment, rules, write_watcher): class Checker(pcls): _NAME = name _RULES = rules _COMMENT = comment _WWATCHER = write_watcher return Checker def RuledContainer(pcls): """ Factory for abstract 'container with rules' class. See ConfigDict for details, examples and tests. """ class _RuledContainer(pcls): RULE_COMMENT = 0 RULE_CHECKER = 1 # Reserved ... RULE_DEFAULT = -1 RULE_CHECK_MAP = { bool: validators.BoolCheck, 'bin': validators.NotUnicode, 'bool': validators.BoolCheck, 'b36': validators.B36Check, 'dir': validators.DirCheck, 'directory': validators.DirCheck, 'ignore': validators.IgnoreCheck, 'email': validators.EmailCheck, 'False': False, 'false': False, 'file': validators.FileCheck, 'float': float, 'gpgkeyid': validators.GPGKeyCheck, 'hostname': validators.HostNameCheck, 'int': int, 'long': long, 'multiline': unicode, 'new file': validators.NewPathCheck, 'new dir': validators.NewPathCheck, 'new directory': validators.NewPathCheck, 'path': validators.PathCheck, str: unicode, 'slashslug': validators.SlashSlugCheck, 'slug': validators.SlugCheck, 'str': unicode, 'True': True, 'true': True, 'timestamp': long, 'unicode': unicode, 'url': validators.UrlCheck, # FIXME: check more than the scheme? 'webroot': validators.WebRootCheck } def _default_write_watcher(self, *args): self._changed = True _NAME = 'container' _RULES = None _COMMENT = None _MAGIC = True _WWATCHER = _default_write_watcher def __init__(self, *args, **kwargs): rules = kwargs.get('_rules', self._RULES or {}) self._name = kwargs.get('_name', self._NAME) self._comment = kwargs.get('_comment', self._COMMENT) self._write_watcher = kwargs.get('_write_watcher', self._WWATCHER) enable_magic = kwargs.get('_magic', self._MAGIC) for kw in ('_rules', '_comment', '_name', '_magic', '_write_watcher'): if kw in kwargs: del kwargs[kw] pcls.__init__(self) self._key = self._name self._rules_source = rules self._changed = False self.rules = {} self.set_rules(rules) self.update(*args, **kwargs) self._magic = enable_magic # Enable the getitem/getattr magic def __str__(self): return json.dumps(self, sort_keys=True, indent=2) def __unicode__(self): return json.dumps(self, sort_keys=True, indent=2) def as_config_bytes(self, _type=None, _xtype=None): of = io.BytesIO() self.as_config(_type=_type, _xtype=_xtype).write(of) return of.getvalue() def key_types(self, key): if key not in self.rules: key = '_any' if key in self.rules and hasattr(self.rules[key], '_types'): return self.rules[key]._types else: return [] def as_config(self, config=None, _type=None, _xtype=None): config = config or CommentedEscapedConfigParser() section = self._name if self._comment: section += ': %s' % self._comment added_section = False keys = self.rules.keys() if _type: keys = [k for k in keys if _type in self.key_types(k)] ignore = self.ignored_keys() | set(['_any']) if not _type: if not keys or '_any' in keys: keys.extend(self.keys()) keys = [k for k in sorted(set(keys)) if k not in ignore] set_keys = set(self.keys()) for key in keys: if not hasattr(self[key], 'as_config'): if key in self.rules: comment = self.rules[key][self.RULE_COMMENT] else: comment = '' value = self[key] if value is not None and value != '': if key not in set_keys: key = ';' + key comment = '(default) ' + comment if not added_section: config.add_section(str(section)) added_section = True if _xtype not in self.key_types(key) or not _xtype: config.set(section, key, value, comment) for key in keys: if hasattr(self[key], 'as_config'): if isinstance(self[key], list): # If a list is marked public, we export all items self[key].as_config(config=config) else: self[key].as_config( config=config, _type=_type, _xtype=_xtype) return config def reset(self, rules=True, data=True): raise Exception(_('Please override this method')) def set_rules(self, rules): safe_assert(isinstance(rules, dict)) self.reset() for key, rule in rules.iteritems(): self.add_rule(key, rule) def add_rule(self, key, rule): if not ((isinstance(rule, (list, tuple))) and (key == CleanText(key, banned=CleanText.NONVARS).clean) and (not self.real_hasattr(key))): raise TypeError('add_rule(%s, %s): Bad key or rule.' % (key, rule)) orule, rule = rule, ConfigRule(*rule[:]) if hasattr(orule, '_types'): rule._types = orule._types self.rules[key] = rule check = rule[self.RULE_CHECKER] try: check = self.RULE_CHECK_MAP.get(check, check) rule[self.RULE_CHECKER] = check except TypeError: pass name = '%s/%s' % (self._name, key) comment = rule[self.RULE_COMMENT] value = rule[self.RULE_DEFAULT] ww = self.real_getattr('_write_watcher') if (isinstance(check, dict) and value is not None and not isinstance(value, (dict, list))): raise TypeError(_('Only lists or dictionaries can contain ' 'dictionary values (key %s).') % name) if isinstance(value, dict) and check is False: pcls.__setitem__(self, key, ConfigDict(_name=name, _comment=comment, _write_watcher=ww, _rules=value)) elif isinstance(value, dict): if value: raise ConfigValueError(_('Subsections must be immutable ' '(key %s).') % name) sub_rule = {'_any': [rule[self.RULE_COMMENT], check, None]} checker = _MakeCheck(ConfigDict, name, check, sub_rule, ww) pcls.__setitem__(self, key, checker()) rule[self.RULE_CHECKER] = checker elif isinstance(value, list): if value: raise ConfigValueError(_('Lists cannot have default ' 'values (key %s).') % name) sub_rule = {'_any': [rule[self.RULE_COMMENT], check, None]} checker = _MakeCheck(ConfigList, name, comment, sub_rule, ww) pcls.__setitem__(self, key, checker()) rule[self.RULE_CHECKER] = checker elif not isinstance(value, (type(None), int, long, bool, float, str, unicode)): raise TypeError(_('Invalid type "%s" for key "%s" (value: %s)' ) % (type(value), name, repr(value))) def __fixkey__(self, key): return key.lower() def fmt_key(self, key): return key.lower() def get_rule(self, key): key = self.__fixkey__(key) rule = self.rules.get(key, None) if rule is None: if '_any' in self.rules: rule = self.rules['_any'] else: raise InvalidKeyError(_('Invalid key for %s: %s' ) % (self._name, key)) if isinstance(rule[self.RULE_CHECKER], dict): rule = rule[:] rule[self.RULE_CHECKER] = _MakeCheck( ConfigDict, '%s/%s' % (self._name, key), rule[self.RULE_COMMENT], rule[self.RULE_CHECKER], self._write_watcher) return rule def ignored_keys(self): return set([k for k in self.rules if self.rules[k][self.RULE_CHECKER] == validators.IgnoreCheck]) def walk(self, path, parent=0, key_types=None): if '.' in path: sep = '.' else: sep = '/' path_parts = path.split(sep) cfg = self if parent: vlist = path_parts[-parent:] path_parts[-parent:] = [] else: vlist = [] for part in path_parts: if key_types is not None: if [t for t in cfg.key_types(part) if t not in key_types]: raise AccessError(_('Access denied to %s') % part) cfg = cfg[part] if parent: return tuple([cfg] + vlist) else: return cfg def get(self, key, default=None): key = self.__fixkey__(key) if key in self: return pcls.__getitem__(self, key) if default is None and key in self.rules: return self.rules[key][self.RULE_DEFAULT] return default def __getitem__(self, key): key = self.__fixkey__(key) if key in self.rules or '_any' in self.rules: return self.get(key) return pcls.__getitem__(self, key) def real_getattr(self, attr): try: return pcls.__getattribute__(self, attr) except AttributeError: return False def real_hasattr(self, attr): try: pcls.__getattribute__(self, attr) return True except AttributeError: return False def real_setattr(self, attr, value): return pcls.__setattr__(self, attr, value) def __getattr__(self, attr, default=None): if self.real_hasattr(attr) or not self.real_getattr('_magic'): return pcls.__getattribute__(self, attr) return self[attr] def __setattr__(self, attr, value): if self.real_hasattr(attr) or not self.real_getattr('_magic'): return self.real_setattr(attr, value) self.__setitem__(attr, value) def __passkey__(self, key, value): if hasattr(value, '__passkey__'): value._key = key value._name = '%s/%s' % (self._name, key) def __passkey_recurse__(self, key, value): if hasattr(value, '__passkey__'): if isinstance(value, (list, tuple)): for k in range(0, len(value)): value.__passkey__(value.__fixkey__(k), value[k]) elif isinstance(value, dict): for k in value: value.__passkey__(value.__fixkey__(k), value[k]) def __createkey_and_setitem__(self, key, value): pcls.__setitem__(self, key, value) def __setitem__(self, key, value): key = self.__fixkey__(key) checker = self.get_rule(key)[self.RULE_CHECKER] if not checker is True: if checker is False: if isinstance(value, dict) and isinstance(self[key], dict): for k, v in value.iteritems(): self[key][k] = v return raise ConfigValueError(_('Modifying %s/%s is not ' 'allowed') % (self._name, key)) elif isinstance(checker, (list, set, tuple)): if value not in checker: raise ConfigValueError(_('Invalid value for %s/%s: %s' ) % (self._name, key, value)) elif isinstance(checker, (type, type(RuledContainer))): try: if value is None: value = checker() else: value = checker(value) except (ConfigValueError): raise except (validators.IgnoreValue): return except (ValueError, TypeError): raise ValueError(_('Invalid value for %s/%s: %s' ) % (self._name, key, value)) else: raise Exception(_('Unknown constraint for %s/%s: %s' ) % (self._name, key, checker)) write_watcher = self.real_getattr('_write_watcher') if write_watcher is not None: write_watcher(self, key, value) self.__passkey__(key, value) self.__createkey_and_setitem__(key, value) self.__passkey_recurse__(key, value) def extend(self, src): for val in src: self.append(val) def __iadd__(self, src): self.extend(src) return self return _RuledContainer class ConfigList(RuledContainer(list)): """ A sanity-checking, self-documenting list of program settings. Instances of this class are usually contained within a ConfigDict. >>> lst = ConfigList(_rules={'_any': ['We only like ints', int, 0]}) >>> lst.append('1') '0' >>> lst.extend([2, '3']) >>> lst [1, 2, 3] >>> lst += ['1', '2'] >>> lst [1, 2, 3, 1, 2] >>> lst.extend(range(0, 100)) >>> lst['c'] == lst[int('c', 36)] True """ def reset(self, rules=True, data=True): if rules: self.rules = {} if data: self[:] = [] def __createkey_and_setitem__(self, key, value): while key > len(self): self.append(self.rules['_any'][self.RULE_DEFAULT]) if key == len(self): self.append(value) else: list.__setitem__(self, key, value) def append(self, value): list.append(self, None) try: self[len(self) - 1] = value return b36(len(self) - 1).lower() except: self[len(self) - 1:] = [] raise def __passkey__(self, key, value): if hasattr(value, '__passkey__'): key = b36(key).lower() value._key = key value._name = '%s/%s' % (self._name, key) def __fixkey__(self, key): if isinstance(key, (str, unicode)): try: key = int(key, 36) except ValueError: pass return key def get(self, key, default=None): try: return list.__getitem__(self, self.__fixkey__(key)) except IndexError: return default def __getitem__(self, key): return list.__getitem__(self, self.__fixkey__(key)) def fmt_key(self, key): f = b36(self.__fixkey__(key)).lower() return ('0000' + f)[-4:] if (len(f) < 4) else f def iterkeys(self): return (self.fmt_key(i) for i in range(0, len(self))) def iteritems(self): for k in self.iterkeys(): yield (k, self[k]) def keys(self): return list(self.iterkeys()) def all_keys(self): return list(self.iterkeys()) def values(self): return self[:] def update(self, *args): for l in args: l = list(l) for i in range(0, len(self)): self[i] = l[i] for i in range(len(self), len(l)): self.append(l[i]) class ConfigDict(RuledContainer(dict)): """ A sanity-checking, self-documenting dictionary of program settings. The object must be initialized with a dictionary which describes in a structured way what variables exist, what their legal values are, and what their defaults are and what they are for. Each variable definition expects three values: 1. A human readable description of what the variable is 2. A data type / sanity check 3. A default value If the sanity check is itself a dictionary of rules, values are expected to be dictionaries or lists of items that match the rules defined. This should be used with an empty list or dictionary as a default value. Configuration data can be nested by including a dictionary of further rules in place of the default value. If the default value is an empty list, it is assumed to be a list of values of the type specified. Examples: >>> pot = ConfigDict(_rules={'potatoes': ['How many potatoes?', 'int', 0], ... 'carrots': ['How many carrots?', int, 99], ... 'liquids': ['Fluids we like', False, { ... 'water': ['Liters', int, 0], ... 'vodka': ['Liters', int, 12] ... }], ... 'tags': ['Tags', {'c': ['C', int, 0], ... 'x': ['X', str, '']}, []], ... 'colors': ['Colors', ('red', 'blue'), []]}) >>> sorted(pot.keys()), sorted(pot.values()) (['colors', 'liquids', 'tags'], [[], [], {}]) >>> pot['potatoes'] = pot['liquids']['vodka'] = "123" >>> pot['potatoes'] 123 >>> pot['liquids']['vodka'] 123 >>> pot['carrots'] 99 >>> pot.walk('liquids.vodka') 123 >>> pot.walk('liquids/vodka', parent=True) ({...}, 'vodka') >>> pot['colors'].append('red') '0' >>> pot['colors'].extend(['blue', 'red', 'red']) >>> pot['colors'] ['red', 'blue', 'red', 'red'] >>> pot['tags'].append({'c': '123', 'x': 'woots'}) '0' >>> pot['tags'][0]['c'] 123 >>> pot['tags'].append({'z': 'invalid'}) Traceback (most recent call last): ... ValueError: Invalid value for config/tags/1: ... >>> pot['evil'] = 123 Traceback (most recent call last): ... InvalidKeyError: Invalid key for config: evil >>> pot['liquids']['evil'] = 123 Traceback (most recent call last): ... InvalidKeyError: Invalid key for config/liquids: evil >>> pot['potatoes'] = "moo" Traceback (most recent call last): ... ValueError: Invalid value for config/potatoes: moo >>> pot['colors'].append('green') Traceback (most recent call last): ... ConfigValueError: Invalid value for config/colors/4: green >>> pot.rules['potatoes'] ['How many potatoes?', , 0] >>> isinstance(pot['liquids'], ConfigDict) True """ _NAME = 'config' def reset(self, rules=True, data=True): if rules: self.rules = {} if data: for key in self.keys(): if hasattr(self[key], 'reset'): self[key].reset(rules=rules, data=data) else: dict.__delitem__(self, key) def all_keys(self): return list(set(self.keys()) | set(self.rules.keys()) - self.ignored_keys() - set(['_any'])) def append(self, value): """Add to the dict using an autoselected key""" if '_any' in self.rules: k = b36(max([int(k, 36) for k in self.keys()] + [-1]) + 1).lower() self[k] = value return k else: raise UsageError(_('Cannot append to fixed dict')) def update(self, *args, **kwargs): """Reimplement update, so it goes through our sanity checks.""" for src in args: if hasattr(src, 'keys'): for key in src: self[key] = src[key] else: for key, val in src: self[key] = val for key in kwargs: self[key] = kwargs[key] def parse_config(self, session, data, source='internal'): """ Parse a config file fragment. Invalid data will be ignored, but will generate warnings in the session UI. Returns True on a clean parse, False if any of the settings were bogus. >>> cfg.parse_config(session, '[config/sys]\\nfd_cache_size = 123\\n') True >>> cfg.sys.fd_cache_size 123 >>> cfg.parse_config(session, '[config/bogus]\\nblabla = bla\\n') False >>> [l[1] for l in session.ui.log_buffer if 'bogus' in l[1]][0] 'Invalid (internal): section config/bogus does not exist' >>> cfg.parse_config(session, '[config/sys]\\nhistory_length = 321\\n' ... 'bogus_variable = 456\\n') False >>> cfg.sys.history_length 321 >>> [l[1] for l in session.ui.log_buffer if 'bogus_var' in l[1]][0] u'Invalid (internal): section config/sys, ... """ parser = CommentedEscapedConfigParser() parser.readfp(io.BytesIO(str(data))) def item_sorter(i): try: return (int(i[0], 36), i[1]) except (ValueError, IndexError, KeyError, TypeError): return i all_okay = True for section in parser.sections(): okay = True cfgpath = section.split(':')[0].split('/')[1:] cfg = self added_parts = [] for part in cfgpath: if cfg.fmt_key(part) in cfg.keys(): cfg = cfg[part] elif '_any' in cfg.rules: cfg[part] = {} cfg = cfg[part] else: if session: msg = _('Invalid (%s): section %s does not ' 'exist') % (source, section) session.ui.warning(msg) all_okay = okay = False items = parser.items(section) if okay else [] items.sort(key=item_sorter) for var, val in items: try: cfg[var] = val except (ValueError, KeyError, IndexError): if session: msg = _(u'Invalid (%s): section %s, variable %s=%s' ) % (source, section, var, val) session.ui.warning(msg) all_okay = okay = False return all_okay class PathDict(ConfigDict): _RULES = { '_any': ['Data directory', 'directory', ''] } if __name__ == "__main__": import doctest import sys import copy import mailpile.config.defaults import mailpile.ui rules = copy.deepcopy(mailpile.config.defaults.CONFIG_RULES) rules.update({ 'nest1': ['Nest1', { 'nest2': ['Nest2', str, []], 'nest3': ['Nest3', { 'nest4': ['Nest4', str, []] }, []], }, {}] }) cfg = ConfigDict(_rules=rules) session = mailpile.ui.Session(cfg) session.ui = mailpile.ui.SilentInteraction(cfg) session.ui.block() result = doctest.testmod(optionflags=doctest.ELLIPSIS) print('%s' % (result, )) if result.failed: sys.exit(1) ================================================ FILE: mailpile/config/defaults.py ================================================ from __future__ import print_function APPVER = "1.0.0rc6" ABOUT = """\ Mailpile.py a tool Copyright 2013-2018, Mailpile ehf v%8.0008s for searching and organizing piles of e-mail This program is free software: you can redistribute it and/or modify it under the terms of either the GNU Affero General Public License as published by the Free Software Foundation. See the file COPYING.md for details. """ % APPVER ############################################################################# import os import sys import time from mailpile.config.base import PathDict from mailpile.config.base import ConfigRule as c from mailpile.config.base import CriticalConfigRule as X from mailpile.config.base import PublicConfigRule as p from mailpile.config.base import KeyConfigRule as k _ = lambda string: string DEFAULT_SENDMAIL = '|/usr/sbin/sendmail -i %(rcpt)s' CONFIG_PLUGINS = [] CONFIG_RULES = { 'version': p(_('Mailpile program version'), str, APPVER), 'homedir': p(_('Location of Mailpile data'), False, '(unset)'), 'timestamp': [_('Configuration timestamp'), int, int(time.time())], 'master_key': k(_('Master symmetric encryption key'), str, ''), 'sys': p(_('Technical system settings'), False, { 'fd_cache_size': p(_('Max files kept open at once'), int, 500), 'minfree_mb': p(_('Required free disk space (MB)'), int, 1024), 'history_length': (_('History length (lines, <0=no save)'), int, 100), 'http_host': p(_('Listening host for web UI'), 'hostname', 'localhost'), 'http_port': p(_('Listening port for web UI'), int, 33411), 'http_path': p(_('HTTP path of web UI'), 'webroot', ''), 'http_no_auth': X(_('Disable HTTP authentication'), bool, False), 'ajax_timeout': (_('AJAX Request timeout'), int, 10000), 'postinglist_kb': (_('Posting list target size in KB'), int, 64), 'sort_max': (_('Max results we sort "well"'), int, 2500), 'snippet_max': (_('Max length of metadata snippets'), int, 275), 'debug': p(_('Debugging flags'), str, ''), 'experiments': (_('Enabled experiments'), str, ''), 'gpg_keyserver': (_('Host:port of PGP keyserver'), str, 'pool.sks-keyservers.net'), 'gpg_home': p(_('Override the home directory of GnuPG'), 'dir', None), 'gpg_binary': p(_('Override the default GPG binary path'), 'file', None), 'local_mailbox_id': (_('Local read/write Maildir'), 'b36', ''), 'mailindex_file': (_('Metadata index file'), 'file', ''), 'postinglist_dir': (_('Search index directory'), 'dir', ''), 'mailbox': [_('Mailboxes we index'), 'bin', []], 'plugins_early': p(_('Plugins to load before login'), CONFIG_PLUGINS, []), 'plugins': [_('Plugins to load after login'), CONFIG_PLUGINS, []], 'path': [_('Locations of assorted data'), False, { 'html_theme': [_('User interface theme'), 'dir', 'default-theme'], 'vcards': [_('Location of vCards'), 'dir', 'vcards'], 'event_log': [_('Location of event log'), 'dir', 'logs'], }], 'lockdown': p(_('Demo mode, disallow changes'), str, ''), 'login_banner': p(_('A custom banner for the login page'), str, ''), 'proxy': p(_('Proxy settings'), False, { 'protocol': p(_('Proxy protocol'), ["tor", "tor-risky", "socks5", "socks4", "http", "none", "system", "unknown"], "system"), 'fallback': p(_('Allow fallback to direct conns'), bool, False), 'username': (_('User name'), str, ''), 'password': (_('Password'), str, ''), 'host': p(_('Host'), str, ''), 'port': p(_('Port'), int, 8080), 'no_proxy': p(_('List of hosts to avoid proxying'), str, 'localhost, 127.0.0.1, ::1') }), 'tor': p(_('Tor settings'), False, { 'binary': p(_('Override the default Tor binary path'), 'file', None), 'systemwide':p(_('Use shared system-wide Tor (not our own)'), bool, True), 'socks_host':p(_('Socks host'), str, ''), 'socks_port':p(_('Socks Port'), int, 0), 'ctrl_port': p(_('Control Port'), int, 0), 'ctrl_auth': p(_('Control Password'), str, '') }) }), 'prefs': p(_("User preferences"), False, { 'num_results': (_('Search results per page'), int, 20), 'rescan_interval': (_('Misc. data refresh frequency'), int, 900), 'open_in_browser': p(_('Open in browser on startup'), bool, True), 'auto_mark_as_read':(_('Automatically mark e-mail as read'), bool, True), 'web_content': (_('Download content from the web'), ["off", "anon", "on"], "unknown"), 'html5_sandbox': (_('Use HTML5 sandboxes'), bool, True), 'attachment_urls': (_('URLs to treat as attachments (regex)'), str, []), 'weak_crypto_max_age': ( _('Accept weak crypto in messages older than this (unix time)'), int, 0), 'encrypted_block_html': (_('Never display HTML from encrypted mail'), bool, True), 'encrypted_block_web': (_('Never fetch web content from encrypted mail'), bool, True), 'gpg_use_agent': (_('Use the local GnuPG agent'), bool, False), 'gpg_clearsign': X(_('Inline PGP signatures or attached'), bool, False), 'gpg_recipient': (_('Encrypt local data to ...'), 'gpgkeyid', ''), 'gpg_email_key': (_('Enable e-mail based public key distribution'), bool, True), 'gpg_html_wrap': (_('Wrap keys and signatures in helpful HTML'), bool, True), 'antiphishing': (_("Enable experimental anti-phishing heuristics "), bool, False), 'key_tofu': (_("Key Import Behaviour"), False, { 'autocrypt': (_('Auto-import keys using Autocrypt state machine'), bool, True), 'historic': (_('Auto-import keys using communication history'), bool, True), 'hist_min': (_('Require this many signed- or encrypted e-mails'), int, 3), 'hist_recent': (_('Consider the most recent N e-mails (per sender)'), int, 6), 'hist_origins':(_('Origins to auto-import keys from (historic)'), str, 'e-mail, wkd, koo'), 'min_interval': (_('Interval between TOFU checks (per sender)'), int, 1800), }), 'key_trust': (_("Key Trust Model"), False, { 'threshold': (_('Minimum number of signatures required'), int, 5), 'window_days': (_('Window of time (days) to evaluate trust'), int, 180), 'sig_warn_pct': (_('Signed ratio (%) above which we expect sigs'), int, 80), 'key_trust_pct':(_('Ratio of key use (%) above which we trust key'), int, 90), 'key_new_pct': (_('Consider key new below this ratio (%) of sigs'), int, 10) }), 'openpgp_header': X(_('Advertise PGP preferences in a header?'), ['', 'sign', 'encrypt', 'signencrypt'], 'signencrypt'), 'crypto_policy': X(_('Default encryption policy for outgoing mail'), str, 'none'), 'inline_pgp': (_('Use inline PGP when possible'), bool, True), 'encrypt_subject': (_('Encrypt subjects by default'), bool, True), 'default_order': (_('Default sort order'), str, 'rev-date'), 'obfuscate_index':X(_('Key to use to scramble the index'), str, ''), 'index_encrypted':X(_('Make encrypted content searchable'), bool, False), 'encrypt_mail': X(_('Encrypt locally stored mail'), bool, False), 'encrypt_index': X(_('Encrypt the local search index'), bool, False), 'encrypt_vcards': X(_('Encrypt the contact database'), bool, True), 'encrypt_events': X(_('Encrypt the event log'), bool, True), 'encrypt_misc': X(_('Encrypt misc. local data'), bool, True), 'allow_deletion': X(_('Allow permanent deletion of e-mails'), bool, False), 'deletion_ratio': X(_('Max fraction of source mail to delete per pass'), float, 0.75), # FIXME: # 'backup_to_web': X(_('Backup settings and keys to mobile web app'), # bool, True), # 'backup_to_email':X(_('Backup settings and keys to e-mail'), str, ''), 'rescan_command': (_('Command run before rescanning'), str, ''), 'default_email': (_('Default outgoing e-mail address'), 'email', ''), 'default_route': (_('Default outgoing mail route'), str, ''), 'line_length': (_('Target line length, <40 disables reflow'), int, 65), 'always_bcc_self': (_('Always BCC self on outgoing mail'), bool, True), 'default_messageroute': (_('Default outgoing mail route'), str, ''), 'language': p(_('User interface language'), str, ''), 'vcard': [_("vCard import/export settings"), False, { 'importers': [_("vCard import settings"), False, {}], 'exporters': [_("vCard export settings"), False, {}], 'context': [_("vCard context helper settings"), False, {}], }], 'friendly_pipes': (_("Enable sh-like pipes in the CLI"), bool, True), }), 'web': (_("Web Interface Preferences"), False, { 'keybindings': (_('Enable keyboard short-cuts'), bool, False), 'developer_mode': (_('Enable developer-only features'), bool, False), 'friendly_dates': (_('UI uses "friendly" date/times'), bool, True), 'setup_complete': (_('User completed setup experience'), bool, False), 'display_density': (_('Display density of interface'), str, 'comfy'), 'quoted_reply': (_('Quote replies to messages'), str, 'unset'), 'nag_backup_key': (_('Nag user to backup their key'), int, 0), 'subtags_collapsed': (_('Collapsed subtags in sidebar'), str, []), 'donate_visibility': (_('Display donate link in topbar?'), bool, True), 'email_html_hint': (_('Display HTML hints?'), bool, True), 'email_crypto_hint': (_('Display crypto hints?'), bool, True), 'email_reply_hint': (_('Display reply hints?'), bool, True), 'email_tag_hint': (_('Display tagging hints?'), bool, True), 'release_notes': (_('Display release notes?'), bool, True) }), 'logins': [_('Credentials allowed to access Mailpile'), { 'password': (_('Salted and hashed password'), str, '') }, {}], 'secrets': [_('Secrets the user wants saved'), { 'password': (_('A secret'), str, ''), 'policy': (_('Security policy'), ["store", "cache-only", "fail", "protect"], 'store') }, {}], 'tls': [_('Settings for TLS certificate validation'), { 'server': (_('Server hostname:port'), str, ''), 'accept_certs': (_('SHA256 of acceptable certs'), str, []), 'use_web_ca': (_('Use web certificate authorities'), bool, True) }, {}], 'routes': [_('Outgoing message routes'), { 'name': (_('Route name'), str, ''), 'protocol': (_('Messaging protocol'), ["smtp", "smtptls", "smtpssl", "local"], 'smtp'), 'username': (_('User name'), str, ''), 'password': (_('Password'), str, ''), 'auth_type': (_('Authentication scheme'), str, 'password-cleartext'), 'command': (_('Shell command'), str, ''), 'host': (_('Host'), str, ''), 'port': (_('Port'), int, 587) }, {}], 'sources': [_('Incoming message sources'), { 'name': (_('Source name'), str, ''), 'profile': (_('Profile this source belongs to'), str, ''), 'enabled': (_('Is this mail source enabled?'), bool, True), 'protocol': (_('Mail source protocol'), ["local", "imap", "imap_ssl", "imap_tls", "pop3", "pop3_ssl", # These are all obsolete, handled as local: "mbox", "maildir", "macmaildir", "gmvault"], ''), 'pre_command': (_('Shell command run before syncing'), str, ''), 'post_command': (_('Shell command run after syncing'), str, ''), 'interval': (_('How frequently to check for mail'), int, 300), 'username': (_('User name'), str, ''), 'password': (_('Password'), str, ''), 'auth_type': (_('Authentication scheme'), str, 'password'), 'host': (_('Host'), str, ''), 'port': (_('Port'), int, 993), 'keepalive': (_('Keep server connections alive'), bool, False), 'discovery': (_('Mailbox discovery policy'), False, { 'paths': (_('Paths to watch for new mailboxes'), 'bin', []), 'policy': (_('Default mailbox policy'), ['unknown', 'ignore', 'watch', 'read', 'move', 'sync'], 'unknown'), 'local_copy': (_('Copy mail to a local mailbox?'), bool, False), 'parent_tag': (_('Parent tag for mailbox tags'), str, '!CREATE'), 'guess_tags': (_('Guess which local tags match'), bool, True), 'create_tag': (_('Create a tag for each mailbox?'), bool, True), 'visible_tags':(_('Make tags visible by default?'), bool, True), 'process_new': (_('Is a potential source of new mail'), bool, True), 'apply_tags': (_('Tags applied to messages'), str, []), 'max_mailboxes':(_('Max mailboxes to add'), int, 100), }), 'mailbox': (_('Mailboxes'), { 'name': (_('The name of this mailbox'), str, ''), 'path': (_('Mailbox source path'), str, ''), 'policy': (_('Mailbox policy'), ['unknown', 'ignore', 'read', 'move', 'sync', 'inherit'], 'inherit'), 'local': (_('Local mailbox path'), 'bin', ''), 'process_new': (_('Is a source of new mail'), bool, True), 'primary_tag': (_('A tag representing this mailbox'), str, ''), 'apply_tags': (_('Tags applied to messages'), str, []), }, {}) }, {}] } if __name__ == "__main__": import mailpile.config.defaults from mailpile.config.base import ConfigDict print('%s' % (ConfigDict(_name='mailpile', _comment='Base configuration', _rules=mailpile.config.defaults.CONFIG_RULES ).as_config_bytes(), )) ================================================ FILE: mailpile/config/detect.py ================================================ try: import ssl except ImportError: ssl = None try: import sockschain as socks except ImportError: try: import socks except ImportError: socks = None ================================================ FILE: mailpile/config/manager.py ================================================ from __future__ import print_function import copy import cPickle import io import jinja2 import json import os import socket import sys import random import re import threading import fasteners import traceback import ConfigParser import errno from urllib import quote, unquote, getproxies from urlparse import urlparse try: from appdirs import AppDirs except ImportError: AppDirs = None import mailpile.platforms from mailpile.command_cache import CommandCache from mailpile.crypto.streamer import DecryptingStreamer from mailpile.crypto.gpgi import GnuPG from mailpile.eventlog import EventLog, Event, GetThreadEvent from mailpile.httpd import HttpWorker from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import OpenMailbox, NoSuchMailboxError, wervd from mailpile.mailutils import FormatMbxId, MBX_ID_LEN from mailpile.search import MailIndex from mailpile.search_history import SearchHistory from mailpile.security import SecurePassphraseStorage from mailpile.ui import Session, BackgroundInteraction from mailpile.util import * from mailpile.vcard import VCardStore from mailpile.vfs import vfs, FilePath, MailpileVfsRoot from mailpile.workers import Worker, ImportantWorker, DumbWorker, Cron import mailpile.i18n import mailpile.security import mailpile.util import mailpile.vfs from mailpile.config.base import * from mailpile.config.paths import DEFAULT_WORKDIR, DEFAULT_SHARED_DATADIR from mailpile.config.paths import LOCK_PATHS from mailpile.config.defaults import APPVER from mailpile.config.detect import socks from mailpile.www.jinjaloader import MailpileJinjaLoader MAX_CACHED_MBOXES = 5 GLOBAL_INDEX_CHECK = ConfigLock() GLOBAL_INDEX_CHECK.acquire() class ConfigManager(ConfigDict): """ This class manages the live global mailpile configuration. This includes the settings themselves, as well as global objects like the index and references to any background worker threads. """ def __init__(self, workdir=None, shareddatadir=None, rules={}): ConfigDict.__init__(self, _rules=rules, _magic=False) self.workdir = os.path.abspath(workdir or DEFAULT_WORKDIR()) self.gnupghome = None mailpile.vfs.register_alias('/Mailpile', self.workdir) self.shareddatadir = os.path.abspath(shareddatadir or DEFAULT_SHARED_DATADIR()) mailpile.vfs.register_alias('/Share', self.shareddatadir) self.vfs_root = MailpileVfsRoot(self) mailpile.vfs.register_handler(0000, self.vfs_root) self.conffile = os.path.join(self.workdir, 'mailpile.cfg') self.conf_key = os.path.join(self.workdir, 'mailpile.key') self.conf_pub = os.path.join(self.workdir, 'mailpile.rc') # Process lock files are not actually created until the first acquire() self.lock_pubconf, self.lock_workdir = LOCK_PATHS(self.workdir) self.lock_pubconf = fasteners.InterProcessLock(self.lock_pubconf) # If the master key changes, we update the file on save, otherwise # the file is untouched. So we keep track of things here. self._master_key = '' self._master_key_ondisk = None self._master_key_passgen = -1 self.detected_memory_corruption = False # Make sure we have a silent background session self.background = Session(self) self.background.ui = BackgroundInteraction(self) self.plugins = None self.tor_worker = None self.http_worker = None self.dumb_worker = DumbWorker('Dumb worker', self.background) self.slow_worker = self.dumb_worker self.scan_worker = self.dumb_worker self.save_worker = self.dumb_worker self.other_workers = [] self.mail_sources = {} self.event_log = None self.index = None self.index_loading = None self.index_check = GLOBAL_INDEX_CHECK self.vcards = {} self.search_history = SearchHistory() self._mbox_cache = [] self._running = {} self._lock = ConfigRLock() self.loaded_config = False def cache_debug(msg): if self.background and 'cache' in self.sys.debug: self.background.ui.debug(msg) self.command_cache = CommandCache(debug=cache_debug) self.passphrases = { 'DEFAULT': SecurePassphraseStorage(), } self.jinja_env = jinja2.Environment( loader=MailpileJinjaLoader(self), cache_size=400, autoescape=True, trim_blocks=True, extensions=['jinja2.ext.i18n', 'jinja2.ext.with_', 'jinja2.ext.do', 'jinja2.ext.autoescape', 'mailpile.www.jinjaextensions.MailpileCommand'] ) self.cron_schedule = {} self.cron_worker = Cron(self.cron_schedule, 'Cron worker', self.background) self.cron_worker.daemon = True self.cron_worker.start() self._magic = True # Enable the getattr/getitem magic def create_and_lock_workdir(self, session): # Make sure workdir exists and that other processes are not using it. if not os.path.exists(self.workdir): if session: session.ui.notify(_('Creating: %s') % self.workdir) os.makedirs(self.workdir, mode=0o700) mailpile.platforms.RestrictReadAccess(self.workdir) # Once acquired, lock_workdir is only released by process termination. if not isinstance(self.lock_workdir, fasteners.InterProcessLock): ipl = fasteners.InterProcessLock(self.lock_workdir) if ipl.acquire(blocking=False): self.lock_workdir = ipl else: if session: session.ui.error(_('Another Mailpile or program is' ' using the profile directory')) sys.exit(1) def load(self, session, *args, **kwargs): from mailpile.plugins.core import Rescan # This should happen somewhere, may as well happen here. We don't # rely on Python's random for anything important, but it's still # nice to seed it well. random.seed(os.urandom(8)) keep_lockdown = self.sys.lockdown with self._lock: rv = self._unlocked_load(session, *args, **kwargs) if not kwargs.get('public_only'): # If the app version does not match the config, run setup. if self.version != APPVER: from mailpile.plugins.setup_magic import Setup Setup(session, 'setup').run() # Trigger background-loads of everything Rescan(session, 'rescan')._idx(wait=False) # Record where our GnuPG keys live self.gnupghome = GnuPG(self).gnupghome() if keep_lockdown: self.sys.lockdown = keep_lockdown return rv def load_master_key(self, passphrase, _raise=None): keydata = [] if passphrase.is_set(): with open(self.conf_key, 'rb') as fd: hdrs = dict(l.split(': ', 1) for l in fd if ': ' in l) salt = hdrs.get('Salt', '').strip() kdfp = hdrs.get('KDF', '').strip() or None if kdfp: try: kdf, params = kdfp.split(' ', 1) kdfp = {} kdfp[kdf] = json.loads(params) except ValueError: kdfp = {} parser = lambda d: keydata.extend(d) for (method, sps) in passphrase.stretches(salt, params=kdfp): try: with open(self.conf_key, 'rb') as fd: decrypt_and_parse_lines(fd, parser, self, newlines=True, gpgi=GnuPG(self), passphrase=sps) break except IOError: keydata = [] if keydata: self.passphrases['DEFAULT'].copy(passphrase) self.set_master_key(''.join(keydata)) self._master_key_ondisk = self.get_master_key() self._master_key_passgen = self.passphrases['DEFAULT'].generation return True else: if _raise is not None: raise _raise('Failed to decrypt master key') return False def _load_config_lines(self, filename, lines): collector = lambda ll: lines.extend(ll) if os.path.exists(filename): with open(filename, 'rb') as fd: decrypt_and_parse_lines(fd, collector, self) def _discover_plugins(self): # Discover plugins and update the config rule to match from mailpile.plugins import PluginManager self.plugins = PluginManager(config=self, builtin=True).discover([ os.path.join(self.shareddatadir, 'contrib'), os.path.join(self.workdir, 'plugins') ]) self.sys.plugins.rules['_any'][ self.RULE_CHECKER] = [None] + self.plugins.loadable() self.sys.plugins_early.rules['_any'][ self.RULE_CHECKER] = [None] + self.plugins.loadable_early() def _configure_default_plugins(self): if (len(self.sys.plugins) == 0) and self.loaded_config: self.sys.plugins.extend(self.plugins.DEFAULT) for plugin in self.plugins.WANTED: if plugin in self.plugins.available(): self.sys.plugins.append(plugin) else: for pos in self.sys.plugins.keys(): name = self.sys.plugins[pos] if name in self.plugins.RENAMED: self.sys.plugins[pos] = self.plugins.RENAMED[name] def _unlocked_load(self, session, public_only=False): # This method will attempt to load the full configuration. # # The Mailpile configuration is in two parts: # - public data in "mailpile.rc" # - private data in "mailpile.cfg" (encrypted) # # This method may successfully load and process from the public part, # but fail to load the encrypted part due to a lack of authentication. # In this case IOError will be raised. # if not public_only: self.create_and_lock_workdir(session) if session is None: session = self.background if self.index: self.index_check.acquire() self.index = None # Set the homedir default self.rules['homedir'][2] = self.workdir self._rules_source['homedir'][2] = self.workdir self.reset(rules=False, data=True) self.loaded_config = False pub_lines, prv_lines = [], [] try: self._load_config_lines(self.conf_pub, pub_lines) if public_only: return if os.path.exists(self.conf_key): mailpile.platforms.RestrictReadAccess(self.conf_key) self.load_master_key(self.passphrases['DEFAULT'], _raise=IOError) self._load_config_lines(self.conffile, prv_lines) except IOError: self.loaded_config = False raise except (ValueError, OSError): # Bad data in config or config doesn't exist: just forge onwards pass finally: ## The following things happen, no matter how loading went... # Discover plugins first, as this affects what is or is not valid # in the configuration file. self._discover_plugins() # Parse once (silently), to figure out which plugins to load... self.parse_config(None, '\n'.join(pub_lines), source=self.conf_pub) self.parse_config(None, '\n'.join(prv_lines), source=self.conffile) # Enable translations! mailpile.i18n.ActivateTranslation( session, self, self.prefs.language) # Configure and load plugins as per config requests with mailpile.i18n.i18n_disabled: self._configure_default_plugins() self.load_plugins(session) # Now all the plugins are loaded, reset and parse again! self.reset_rules_from_source() self.parse_config(session, '\n'.join(pub_lines), source=self.conf_pub) self.parse_config(session, '\n'.join(prv_lines), source=self.conffile) self._changed = False # Do this again, so renames and cleanups persist self._configure_default_plugins() ## The following events only happen when we've successfully loaded ## both config files! # Open event log dec_key_func = lambda: self.get_master_key() enc_key_func = lambda: (self.prefs.encrypt_events and self.get_master_key()) self.event_log = EventLog(self.data_directory('event_log', mode='rw', mkdir=True), dec_key_func, enc_key_func ).load() if 'log' in self.sys.debug: self.event_log.ui_watch(session.ui) else: self.event_log.ui_unwatch(session.ui) # Configure security module mailpile.security.KNOWN_TLS_HOSTS = self.tls # Load VCards self.vcards = VCardStore(self, self.data_directory('vcards', mode='rw', mkdir=True)) # Recreate VFS root in case new things have been found self.vfs_root.rescan() # Load Search History self.search_history = SearchHistory.Load(self, merge=self.search_history) # OK, we're happy self.loaded_config = True def reset_rules_from_source(self): with self._lock: self.set_rules(self._rules_source) self.sys.plugins.rules['_any'][ self.RULE_CHECKER] = [None] + self.plugins.loadable() self.sys.plugins_early.rules['_any'][ self.RULE_CHECKER] = [None] + self.plugins.loadable_early() def load_plugins(self, session): with self._lock: from mailpile.plugins import PluginManager plugin_list = set(PluginManager.REQUIRED + self.sys.plugins + self.sys.plugins_early) for plugin in plugin_list: if plugin is not None: session.ui.mark(_('Loading plugin: %s') % plugin) self.plugins.load(plugin) session.ui.mark(_('Processing manifests')) self.plugins.process_manifests() self.prepare_workers(session) def save(self, *args, **kwargs): with self._lock: self._unlocked_save(*args, **kwargs) def get_master_key(self): if not self._master_key: return '' k1, k2, k3 = (k[1:] for k in self._master_key) if k1 == k2 == k3: # This is the only result we like! return k1 else: # Hard fail into read-only lockdown. The periodic health # check will notify the user we are broken. self.detected_memory_corruption = True # Try and recover; best 2 out of 3. if k1 in (k2, k3): return self.set_master_key(k1) if k2 in (k1, k3): return self.set_master_key(k2) mailpile.util.QUITTING = True raise IOError("Failed to access master_key") def set_master_key(self, key): # Prefix each key with a unique character to prevent optimization self._master_key = [i + key for i in ('1', '2', '3')] return key def _delete_old_master_keys(self, keyfile): """ We keep old master key files around for up to 5 days, so users can revert if they make some sort of horrible mistake. After that we delete the backups because they're technically a security risk. """ maxage = time.time() - (5 * 24 * 3600) prefix = os.path.basename(keyfile) + '.' dirname = os.path.dirname(keyfile) for f in os.listdir(dirname): fn = os.path.join(dirname, f) if f.startswith(prefix) and (os.stat(fn).st_mtime < maxage): safe_remove(fn) def _save_master_key(self, keyfile): if not self.get_master_key(): return False # We keep the master key in a file of its own... want_renamed_keyfile = None master_passphrase = self.passphrases['DEFAULT'] if (self._master_key_passgen != master_passphrase.generation or self._master_key_ondisk != self.get_master_key()): if os.path.exists(keyfile): want_renamed_keyfile = keyfile + ('.%x' % time.time()) if not want_renamed_keyfile and os.path.exists(keyfile): # Key file exists, nothing needs to be changed. Happy! # Delete any old key backups we have laying around self._delete_old_master_keys(keyfile) return True # Figure out whether we are encrypting to a GPG key, or using # symmetric encryption (with the 'DEFAULT' passphrase). gpgr = self.prefs.get('gpg_recipient', '').replace(',', ' ') tokeys = (gpgr.split() if (gpgr and gpgr not in ('!CREATE', '!PASSWORD')) else None) if not tokeys and not master_passphrase.is_set(): # Without recipients or a passphrase, we cannot save! return False if not tokeys: salt = b64w(os.urandom(32).encode('base64')) else: salt = '' # FIXME: Create event and capture GnuPG state? mps = master_passphrase.stretched(salt) gpg = GnuPG(self, passphrase=mps) status, encrypted_key = gpg.encrypt(self.get_master_key(), tokeys=tokeys) if status == 0: if salt: h, b = encrypted_key.replace('\r', '').split('\n\n', 1) encrypted_key = ('%s\nSalt: %s\nKDF: %s\n\n%s' % (h, salt, mps.is_stretched or 'None', b)) try: with open(keyfile + '.new', 'wb') as fd: fd.write(encrypted_key) mailpile.platforms.RestrictReadAccess(keyfile + '.new') if want_renamed_keyfile: os.rename(keyfile, want_renamed_keyfile) os.rename(keyfile + '.new', keyfile) self._master_key_ondisk = self.get_master_key() self._master_key_passgen = master_passphrase.generation # Delete any old key backups we have laying around self._delete_old_master_keys(keyfile) return True except: if (want_renamed_keyfile and os.path.exists(want_renamed_keyfile)): os.rename(want_renamed_keyfile, keyfile) raise return False def _unlocked_save(self, session=None, force=False): newfile = '%s.new' % self.conffile pubfile = self.conf_pub keyfile = self.conf_key self.create_and_lock_workdir(None) self.timestamp = int(time.time()) if session and self.event_log: if 'log' in self.sys.debug: self.event_log.ui_watch(session.ui) else: self.event_log.ui_unwatch(session.ui) # Save the public config data first # Warn other processes against reading public data during write # But wait for 2 s max so other processes can't block Mailpile. try: locked = self.lock_pubconf.acquire(blocking=True, timeout=2) with open(pubfile, 'wb') as fd: fd.write(self.as_config_bytes(_type='public')) finally: if locked: self.lock_pubconf.release() if not self.loaded_config: return # Save the master key if necessary (and possible) master_key_saved = self._save_master_key(keyfile) # We abort the save here if nothing has changed. if not force and not self._changed: return # Reset our "changed" tracking flag. Any changes that happen # during the subsequent saves will mark us dirty again, since # we can't be sure the changes got written out. self._changed = False # This slight over-complication, is a reaction to segfaults in # Python 2.7.5's fd.write() method. Let's just feed it chunks # of data and hope for the best. :-/ config_bytes = self.as_config_bytes(_xtype='public') config_chunks = (config_bytes[i:i + 4096] for i in range(0, len(config_bytes), 4096)) from mailpile.crypto.streamer import EncryptingStreamer if self.get_master_key() and master_key_saved: subj = self.mailpile_path(self.conffile) with EncryptingStreamer(self.get_master_key(), dir=self.tempfile_dir(), header_data={'subject': subj}, name='Config') as fd: for chunk in config_chunks: fd.write(chunk) fd.save(newfile) else: # This may result in us writing the master key out in the # clear, but that is better than losing data. :-( with open(newfile, 'wb') as fd: for chunk in config_chunks: fd.write(chunk) # Keep the last 5 config files around... just in case. backup_file(self.conffile, backups=5, min_age_delta=900) if mailpile.platforms.RenameCannotOverwrite(): try: # We only do this if we have to; we would rather just # use rename() as it's (more) atomic. os.remove(self.conffile) except (OSError, IOError): pass os.rename(newfile, self.conffile) # If we are shutting down, just stop here. if mailpile.util.QUITTING: return # Enable translations mailpile.i18n.ActivateTranslation(None, self, self.prefs.language) # Recreate VFS root in case new things have been configured self.vfs_root.rescan() # Reconfigure the connection broker from mailpile.conn_brokers import Master as ConnBroker ConnBroker.configure() # Notify workers that things have changed. We do this before # the prepare_workers() below, because we only want to notify # workers that were already running. self._unlocked_notify_workers_config_changed() # Prepare any new workers self.prepare_workers(daemons=self.daemons_started(), changed=True) # Invalidate command cache contents that depend on the config self.command_cache.mark_dirty([u'!config']) def _find_mail_source(self, mbx_id, path=None): if path: path = FilePath(path).raw_fp if path[:5] == '/src:': return self.sources[path[5:].split('/')[0]] if path[:4] == 'src:': return self.sources[path[4:].split('/')[0]] for src in self.sources.values(): # Note: we cannot test 'mbx_id in ...' because of case sensitivity. if src.mailbox[FormatMbxId(mbx_id)] is not None: return src return None def get_mailboxes(self, with_mail_source=None, mail_source_locals=False): try: mailboxes = [(FormatMbxId(k), self.sys.mailbox[k], self._find_mail_source(k)) for k in self.sys.mailbox.keys() if self.sys.mailbox[k] != '/dev/null'] except (AttributeError): # Config not loaded, nothing to see here return [] if with_mail_source is True: mailboxes = [(i, p, s) for i, p, s in mailboxes if s] elif with_mail_source is False: if mail_source_locals: mailboxes = [(i, p, s) for i, p, s in mailboxes if (not s) or (not s.enabled)] else: mailboxes = [(i, p, s) for i, p, s in mailboxes if not s] else: pass # All mailboxes, with or without mail sources if mail_source_locals: for i in range(0, len(mailboxes)): mid, path, src = mailboxes[i] mailboxes[i] = (mid, src and src.mailbox[mid].local or path, src) mailboxes.sort() return mailboxes def is_editable_message(self, msg_info): for ptr in msg_info[MailIndex.MSG_PTRS].split(','): if not self.is_editable_mailbox(ptr[:MBX_ID_LEN]): return False editable = False for tid in msg_info[MailIndex.MSG_TAGS].split(','): try: if self.tags and self.tags[tid].flag_editable: editable = True except (KeyError, AttributeError): pass return editable def is_editable_mailbox(self, mailbox_id): try: mailbox_id = ((mailbox_id is None and -1) or (mailbox_id == '' and -1) or int(mailbox_id, 36)) local_mailbox_id = int(self.sys.get('local_mailbox_id', 'ZZZZZ'), 36) return (mailbox_id == local_mailbox_id) except ValueError: return False def load_pickle(self, pfn, delete_if_corrupt=False): pickle_path = os.path.join(self.workdir, pfn) try: with open(pickle_path, 'rb') as fd: master_key = self.get_master_key() if master_key: from mailpile.crypto.streamer import DecryptingStreamer with DecryptingStreamer(fd, mep_key=master_key, name='load_pickle(%s)' % pfn ) as streamer: data = streamer.read() streamer.verify(_raise=IOError) else: data = fd.read() return cPickle.loads(data) except (cPickle.UnpicklingError, IOError, EOFError, OSError): if delete_if_corrupt: safe_remove(pickle_path) raise IOError('Load/unpickle failed: %s' % pickle_path) def save_pickle(self, obj, pfn, encrypt=True): ppath = os.path.join(self.workdir, pfn) if encrypt and self.get_master_key() and self.prefs.encrypt_misc: from mailpile.crypto.streamer import EncryptingStreamer with EncryptingStreamer(self.get_master_key(), dir=self.tempfile_dir(), header_data={'subject': pfn}, name='save_pickle') as fd: cPickle.dump(obj, fd, protocol=0) fd.save(ppath) else: with open(ppath, 'wb') as fd: cPickle.dump(obj, fd, protocol=0) def _mailbox_info(self, mailbox_id, prefer_local=True): try: with self._lock: mbx_id = FormatMbxId(mailbox_id) mfn = self.sys.mailbox[mbx_id] src = self._find_mail_source(mailbox_id, path=mfn) pfn = 'pickled-mailbox.%s' % mbx_id.lower() if prefer_local and src and src.mailbox[mbx_id] is not None: mfn = src and src.mailbox[mbx_id].local or mfn else: pfn += '-R' except (KeyError, TypeError): traceback.print_exc() raise NoSuchMailboxError(_('No such mailbox: %s') % mailbox_id) return mbx_id, src, FilePath(mfn), pfn def save_mailbox(self, session, pfn, mbox): if pfn is not None: mbox.save(session, to=pfn, pickler=lambda o, f: self.save_pickle(o, f)) def uncache_mailbox(self, session, entry, drop=True, force_save=False): """ Safely remove a mailbox from the cache, saving any state changes to the encrypted pickles. If the mailbox is still in use somewhere in the app (as measured by the Python reference counter), we DON'T remove from cache, to ensure each mailbox is represented by exactly one object at a time. """ pfn, mbx_id = entry[:2] # Don't grab mbox, to not add more refs if pfn: def dropit(l): return [c for c in l if (c[0] != pfn)] else: def dropit(l): return [c for c in l if (c[1] != mbx_id)] with self._lock: mboxes = [c[2] for c in self._mbox_cache if ((c[0] == pfn) if pfn else (c[1] == mbx_id))] if len(mboxes) < 1: # Not found, nothing to do here return # At this point, if the mailbox is not in use, there should be # exactly 2 references to it: in mboxes and self._mbox_cache. # However, sys.getrefcount always returns one extra for itself. if sys.getrefcount(mboxes[0]) > 3: if force_save: self.save_mailbox(session, pfn, mboxes[0]) return # This may be slow, but it has to happen inside the lock # otherwise we run the risk of races. self.save_mailbox(session, pfn, mboxes[0]) if drop: self._mbox_cache = dropit(self._mbox_cache) else: keep2 = self._mbox_cache[-MAX_CACHED_MBOXES:] keep1 = dropit(self._mbox_cache[:-MAX_CACHED_MBOXES]) self._mbox_cache = keep1 + keep2 def cache_mailbox(self, session, pfn, mbx_id, mbox): """ Add a mailbox to the cache, potentially evicting other entries if the cache has grown too large. """ with self._lock: if pfn is not None: self._mbox_cache = [ c for c in self._mbox_cache if c[0] != pfn] elif mbx_id: self._mbox_cache = [ c for c in self._mbox_cache if c[1] != mbx_id] self._mbox_cache.append((pfn, mbx_id, mbox)) flush = self._mbox_cache[:-MAX_CACHED_MBOXES] for entry in flush: pfn, mbx_id = entry[:2] self.save_worker.add_unique_task( session, 'Save mailbox %s/%s (drop=%s)' % (mbx_id, pfn, False), lambda: self.uncache_mailbox(session, entry, drop=False)) def flush_mbox_cache(self, session, clear=True, wait=False): if wait: saver = self.save_worker.do else: saver = self.save_worker.add_task with self._lock: flush = self._mbox_cache[:] for entry in flush: pfn, mbx_id = entry[:2] saver(session, 'Save mailbox %s/%s (drop=%s)' % (mbx_id, pfn, clear), lambda: self.uncache_mailbox(session, entry, drop=clear, force_save=True), unique=True) def find_mboxids_and_sources_by_path(self, *paths): def _au(p): return unicode(p[1:] if (p[:5] == '/src:') else p if (p[:4] == 'src:') else vfs.abspath(p)) abs_paths = dict((_au(p), [p]) for p in paths) with self._lock: for sid, src in self.sources.iteritems(): for mid, info in src.mailbox.iteritems(): umfn = _au(self.sys.mailbox[mid]) if umfn in abs_paths: abs_paths[umfn].append((mid, src)) if info.local: lmfn = _au(info.local) if lmfn in abs_paths: abs_paths[lmfn].append((mid, src)) for mid, mfn in self.sys.mailbox.iteritems(): umfn = _au(mfn) if umfn in abs_paths: if umfn[:4] == u'src:': src = self.sources.get(umfn[4:].split('/')[0]) else: src = None abs_paths[umfn].append((mid, src)) return dict((p[0], p[1]) for p in abs_paths.values() if p[1:]) def open_mailbox_path(self, session, path, register=False, raw_open=False): path = vfs.abspath(path) mbox = mbx_mid = mbx_src = None with self._lock: msmap = self.find_mboxids_and_sources_by_path(unicode(path)) if msmap: mbx_mid, mbx_src = list(msmap.values())[0] if (register or raw_open) and mbx_mid is None: mbox = dict(((i, m) for p, i, m in self._mbox_cache) ).get(path, None) if path.raw_fp.startswith('/src:'): path = FilePath(path.raw_fp[1:]) if mbox: pass elif path.raw_fp.startswith('src:'): msrc_id = path.raw_fp[4:].split('/')[0] msrc = self.mail_sources.get(msrc_id) if msrc: mbox = msrc.open_mailbox(None, path.raw_fp) else: mbox = OpenMailbox(path.raw_fp, self, create=False) if register: mbx_mid = self.sys.mailbox.append(unicode(path)) mbox = None # Force a re-open below elif mbox: # (re)-add to the cache; we need to do this here # because we did the opening ourselves instead of # invoking open_mailbox as below. self.cache_mailbox(session, None, path.raw_fp, mbox) if mbx_mid is not None: mbx_mid = FormatMbxId(mbx_mid) if mbox is None: mbox = self.open_mailbox(session, mbx_mid, prefer_local=True) return (mbx_mid, mbox) elif raw_open and mbox: return (mbx_mid, mbox) raise ValueError('Not found') def open_mailbox(self, session, mailbox_id, prefer_local=True, from_cache=False): mbx_id, src, mfn, pfn = self._mailbox_info(mailbox_id, prefer_local=prefer_local) with self._lock: mbox = dict(((p, m) for p, i, m in self._mbox_cache) ).get(pfn, None) try: if mbox is None: if from_cache: return None if session: session.ui.mark(_('%s: Updating: %s') % (mbx_id, mfn)) mbox = self.load_pickle(pfn, delete_if_corrupt=True) if prefer_local and not mbox.is_local: mbox = None else: mbox.update_toc() except AttributeError: mbox = None except KeyboardInterrupt: raise except IOError: mbox = None except: if self.sys.debug: traceback.print_exc() mbox = None if mbox is None: if session: session.ui.mark(_('%s: Opening: %s (may take a while)' ) % (mbx_id, mfn)) editable = self.is_editable_mailbox(mbx_id) if src is not None: msrc = self.mail_sources.get(src._key) mbox = msrc.open_mailbox(mbx_id, mfn.raw_fp) if msrc else None if mbox is None: mbox = OpenMailbox(mfn.raw_fp, self, create=editable) mbox.editable = editable mbox.is_local = prefer_local # Always set these, they can't be pickled mbox._decryption_key_func = lambda: self.get_master_key() mbox._encryption_key_func = lambda: (self.get_master_key() if self.prefs.encrypt_mail else None) # Finally, re-add to the cache self.cache_mailbox(session, pfn, mbx_id, mbox) return mbox def create_local_mailstore(self, session, name=None): path = os.path.join(self.workdir, 'mail') with self._lock: if name is None: name = '%5.5x' % random.randint(0, 16**5) while os.path.exists(os.path.join(path, name)): name = '%5.5x' % random.randint(0, 16**5) if name != '': if not os.path.exists(path): root_mbx = wervd.MailpileMailbox(path) if name.startswith(path) and '..' not in name: path = name else: path = os.path.join(path, os.path.basename(name)) mbx = wervd.MailpileMailbox(path) mbx._decryption_key_func = lambda: self.get_master_key() mbx._encryption_key_func = lambda: (self.get_master_key() if self.prefs.encrypt_mail else None) return FilePath(path), mbx def open_local_mailbox(self, session): with self._lock: local_id = self.sys.get('local_mailbox_id', None) if local_id is None or local_id == '': mailbox, mbx = self.create_local_mailstore(session, name='') local_id = FormatMbxId(self.sys.mailbox.append(mailbox)) self.sys.local_mailbox_id = local_id else: local_id = FormatMbxId(local_id) return local_id, self.open_mailbox(session, local_id) def get_passphrase(self, keyid, description=None, prompt=None, error=None, no_ask=False, no_cache=False): if not no_cache: keyidL = keyid.lower() for sid in self.secrets.keys(): if sid.endswith(keyidL): secret = self.secrets[sid] if secret.policy == 'always-ask': no_cache = True elif secret.policy == 'fail': return False, None elif secret.policy != 'cache-only': sps = SecurePassphraseStorage(secret.password) return (keyidL, sps) if not no_cache: if keyid in self.passphrases: return (keyid, self.passphrases[keyid]) if keyidL in self.passphrases: return (keyidL, self.passphrases[keyidL]) for fprint in self.passphrases: if fprint.endswith(keyid): return (fprint, self.passphrases[fprint]) if fprint.lower().endswith(keyidL): return (fprint, self.passphrases[fprint]) if not no_ask: # This will either record details to the event of the currently # running command/operation, or register a new event. This does # not work as one might hope if ops cross a thread boundary... ev = GetThreadEvent( create=True, message=prompt or _('Please enter your password'), source=self) details = {'id': keyid} if prompt: details['msg'] = prompt if error: details['err'] = error if description: details['dsc'] = description if 'password_needed' in ev.private_data: ev.private_data['password_needed'].append(details) else: ev.private_data['password_needed'] = [details] ev.data['password_needed'] = True # Post a password request to the event log... self.event_log.log_event(ev) return None, None def get_profile(self, email=None): find = email or self.prefs.get('default_email', None) default_sig = _('Sent using Mailpile, Free Software ' 'from www.mailpile.is') default_profile = { 'name': None, 'email': find, 'messageroute': self.prefs.default_messageroute, 'signature': default_sig, 'crypto_policy': 'none', 'crypto_format': 'none', 'vcard': None } profiles = [] if find: profiles = [self.vcards.get_vcard(find)] if not profiles or not profiles[0]: profiles = self.vcards.find_vcards([], kinds=['profile']) if profiles: profiles.sort(key=lambda k: ((0 if k.route else 1), (-len(k.recent_history())), (-len(k.sources())))) if profiles and profiles[0]: profile = profiles[0] psig = profile.signature proute = profile.route default_profile.update({ 'name': profile.fn, 'email': find or profile.email, 'signature': psig if (psig is not None) else default_sig, 'messageroute': (proute if (proute is not None) else self.prefs.default_messageroute), 'crypto_policy': profile.crypto_policy or 'none', 'crypto_format': profile.crypto_format or 'none', 'vcard': profile }) return default_profile def get_route(self, frm, rcpts=['-t']): if len(rcpts) == 1: if rcpts[0].lower().endswith('.onion'): return {"protocol": "smtorp", "host": rcpts[0].split('@')[-1], "port": 25, "auth_type": "", "username": "", "password": ""} routeid = self.get_profile(frm)['messageroute'] if self.routes[routeid] is not None: return self.routes[routeid] else: raise ValueError(_("Route %s for %s does not exist." ) % (routeid, frm)) def data_directory(self, ftype, mode='rb', mkdir=False): """ Return the path to a data directory for a particular type of file data, optionally creating the directory if it is missing. >>> p = cfg.data_directory('html_theme', mode='r', mkdir=False) >>> p == os.path.abspath('shared-data/default-theme') True """ # This should raise a KeyError if the ftype is unrecognized bpath = self.sys.path.get(ftype) if not bpath.startswith('/'): cpath = os.path.join(self.workdir, bpath) if os.path.exists(cpath) or 'w' in mode: bpath = cpath if mkdir and not os.path.exists(cpath): os.mkdir(cpath) else: bpath = os.path.join(self.shareddatadir, bpath) return os.path.abspath(bpath) def data_file_and_mimetype(self, ftype, fpath, *args, **kwargs): # The theme gets precedence core_path = self.data_directory(ftype, *args, **kwargs) path, mimetype = os.path.join(core_path, fpath), None # If there's still nothing there, check our plugins if not os.path.exists(path): from mailpile.plugins import PluginManager path, mimetype = PluginManager().get_web_asset(fpath, path) if os.path.exists(path): return path, mimetype else: return None, None def history_file(self): return os.path.join(self.workdir, 'history') def mailindex_file(self): return os.path.join(self.workdir, 'mailpile.idx') def mailpile_path(self, path): base = (self.workdir + os.sep).replace(os.sep+os.sep, os.sep) if path.startswith(base): return path[len(base):] rbase = os.path.realpath(base) + os.sep rpath = os.path.realpath(path) if rpath.startswith(rbase): return rpath[len(rbase):] return path def tempfile_dir(self): d = os.path.join(self.workdir, 'tmp') if not os.path.exists(d): os.mkdir(d) return d def clean_tempfile_dir(self): try: td = self.tempfile_dir() files = os.listdir(td) random.shuffle(files) for fn in files: fn = os.path.join(td, fn) if os.path.isfile(fn): safe_remove(fn) except (OSError, IOError): pass def postinglist_dir(self, prefix): d = os.path.join(self.workdir, 'search') if not os.path.exists(d): os.mkdir(d) d = os.path.join(d, prefix and prefix[0] or '_') if not os.path.exists(d): os.mkdir(d) return d def need_more_disk_space(self, required=0, nodefault=False, ratio=1.0): """Returns a path where we need more disk space, None if all is ok.""" if self.detected_memory_corruption: return '/' if not (nodefault and required): required = ratio * max(required, self.sys.minfree_mb * 1024 * 1024) for path in (self.workdir, ): if get_free_disk_bytes(path) < required: return path return None def interruptable_wait_for_lock(self): # This construct allows the user to CTRL-C out of things. delay = 0.01 while self._lock.acquire(False) == False: if mailpile.util.QUITTING: raise KeyboardInterrupt('Quitting') time.sleep(delay) delay = min(1, delay*2) self._lock.release() return self._lock def get_index(self, session): # Note: This is a long-running lock, but having two sets of the # index would really suck and this should only ever happen once. with self.interruptable_wait_for_lock(): if self.index: return self.index self.index_loading = MailIndex(self) self.index_loading.load(session) self.index = self.index_loading self.index_loading = None try: self.index_check.release() except: pass return self.index def get_path_index(self, session, path): """ Get a search index by path (instead of the default), or None if no matching index is found. """ idx = None mi, mbox = self.open_mailbox_path(session, path, raw_open=True) if mbox: idx = mbox.get_index(self, mbx_mid=mi) # Return a sad, boring, empty index. if idx is None: import mailpile.index.base idx = mailpile.index.base.BaseIndex(self) return idx def get_proxy_settings(self): if self.sys.proxy.protocol == 'system': proxy_list = getproxies() for proto in ('socks5', 'socks4', 'http'): for url in proxy_list.values(): if url.lower().startswith(proto+'://'): try: p, host, port = url.replace('/', '').split(':') return { 'protocol': proto, 'fallback': self.sys.proxy.fallback, 'host': host, 'port': int(port), 'no_proxy': self.sys.proxy.no_proxy} except (ValueError, IndexError, KeyError): pass elif self.sys.proxy.protocol in ('tor', 'tor-risky'): if self.tor_worker is not None: return { 'protocol': self.sys.proxy.protocol, 'fallback': self.sys.proxy.fallback, 'host': '127.0.0.1', 'port': self.tor_worker.socks_port, 'no_proxy': self.sys.proxy.no_proxy} return self.sys.proxy def open_file(self, ftype, fpath, mode='rb', mkdir=False): if '..' in fpath: raise ValueError(_('Parent paths are not allowed')) fpath, mt = self.data_file_and_mimetype(ftype, fpath, mode=mode, mkdir=mkdir) if not fpath: raise IOError(2, 'Not Found') return fpath, open(fpath, mode), mt def daemons_started(config, which=None): return ((which or config.save_worker) not in (None, config.dumb_worker)) def get_mail_source(config, src_id, start=False, changed=False): ms_thread = config.mail_sources.get(src_id) if (ms_thread and not ms_thread.isAlive()): ms_thread = None if not ms_thread: from mailpile.mail_source import MailSource src_config = config.sources[src_id] ms_thread = MailSource(config.background, src_config) if start: config.mail_sources[src_id] = ms_thread ms_thread.start() if changed: ms_thread.wake_up() return ms_thread def start_tor_worker(config): from mailpile.conn_brokers import Master as ConnBroker from mailpile.crypto.tor import Tor config.tor_worker = Tor( config=config, session=config.background, callbacks=[lambda c: ConnBroker.configure()]) config.tor_worker.start() return config.tor_worker def prepare_workers(self, *args, **kwargs): with self._lock: return self._unlocked_prepare_workers(*args, **kwargs) def _unlocked_prepare_workers(config, session=None, changed=False, daemons=False, httpd_spec=None): # Set our background UI to something that can log. if session: config.background.ui = BackgroundInteraction( config, log_parent=session.ui) # Tell conn broker that we exist from mailpile.conn_brokers import Master as ConnBroker ConnBroker.set_config(config) if 'connbroker' in config.sys.debug: ConnBroker.debug_callback = lambda msg: config.background.ui.debug(msg) else: ConnBroker.debug_callback = None def start_httpd(sspec=None): sspec = sspec or (config.sys.http_host, config.sys.http_port, config.sys.http_path or '') if sspec[0].lower() != 'disabled' and sspec[1] >= 0: try: if mailpile.platforms.NeedExplicitPortCheck(): try: socket.socket().connect((sspec[0],sspec[1])) port_in_use = True except socket.error: port_in_use = False if port_in_use: raise socket.error(errno.EADDRINUSE) config.http_worker = HttpWorker(config.background, sspec) config.http_worker.start() except socket.error as e: if e[0] == errno.EADDRINUSE: session.ui.error( _('Port %s:%s in use by another Mailpile or program' ) % (sspec[0], sspec[1])) # We may start the HTTPD without the loaded config... if not config.loaded_config: if daemons and not config.http_worker: start_httpd(httpd_spec) return # Start the other workers if daemons: for src_id in config.sources.keys(): try: config.get_mail_source(src_id, start=True, changed=changed) except (ValueError, KeyError): pass should_launch_tor = ((not config.sys.tor.systemwide) and (config.sys.proxy.protocol.startswith('tor'))) if config.tor_worker is None: if should_launch_tor: config.start_tor_worker() elif not should_launch_tor: config.tor_worker.stop_tor() config.tor_worker = None if config.slow_worker == config.dumb_worker: config.slow_worker = Worker('Slow worker', config.background) config.slow_worker.wait_until = lambda: ( (not config.save_worker) or config.save_worker.is_idle()) config.slow_worker.start() if config.scan_worker == config.dumb_worker: config.scan_worker = Worker('Scan worker', config.background) config.slow_worker.wait_until = lambda: ( (not config.save_worker) or config.save_worker.is_idle()) config.scan_worker.start() if config.save_worker == config.dumb_worker: config.save_worker = ImportantWorker('Save worker', config.background) config.save_worker.start() if not config.cron_worker: config.cron_worker = Cron( config.cron_schedule, 'Cron worker', config.background) config.cron_worker.start() if not config.http_worker: start_httpd(httpd_spec) if not config.other_workers: from mailpile.plugins import PluginManager for worker in PluginManager.WORKERS: w = worker(config.background) w.start() config.other_workers.append(w) # Update the cron jobs, if necessary if config.cron_worker and config.event_log: from mailpile.postinglist import GlobalPostingList from mailpile.plugins.core import HealthCheck def gpl_optimize(): if HealthCheck.check(config.background, config): rs_interval = (config.prefs.rescan_interval or 1800) runtime = rs_interval / 10 ratio = 2.0 / (7*24*3600.0 / rs_interval) # Optimize 2x/week config.slow_worker.add_unique_task( config.background, 'Optimize GPL', lambda: GlobalPostingList.Optimize( config.background, config.index, lazy=(not user_probably_asleep()), ratio=ratio, runtime=runtime)) # Schedule periodic rescanning, if requested. rescan_interval = config.prefs.rescan_interval if rescan_interval: def rescan(): from mailpile.plugins.core import Rescan if 'rescan' not in config._running: rsc = Rescan(config.background, 'rescan') rsc.serialize = False config.slow_worker.add_unique_task( config.background, 'Rescan', lambda: rsc.run(slowly=True, cron=True)) gpl_optimize() config.cron_worker.add_task('rescan', rescan_interval, rescan) else: config.cron_worker.add_task('gpl_optimize', 1800, gpl_optimize) def metadata_index_saver(): config.save_worker.add_unique_task( config.background, 'save_metadata_index', lambda: config.index.save_changes()) config.cron_worker.add_task( 'save_metadata_index', 900, metadata_index_saver) def search_history_saver(): config.save_worker.add_unique_task( config.background, 'save_search_history', lambda: config.search_history.save(config)) config.cron_worker.add_task( 'save_search_history', 900, search_history_saver) def refresh_command_cache(): config.scan_worker.add_unique_task( config.background, 'refresh_command_cache', lambda: config.command_cache.refresh( event_log=config.event_log), first=True) config.cron_worker.add_task( 'refresh_command_cache', 5, refresh_command_cache) # Schedule plugin jobs from mailpile.plugins import PluginManager def interval(i): if isinstance(i, (str, unicode)): i = config.walk(i) return int(i) def wrap_fast(func): def wrapped(): return func(config.background) return wrapped def wrap_slow(func): def wrapped(): config.slow_worker.add_unique_task( config.background, job, lambda: func(config.background)) return wrapped for job, (i, f) in PluginManager.FAST_PERIODIC_JOBS.iteritems(): config.cron_worker.add_task(job, interval(i), wrap_fast(f)) for job, (i, f) in PluginManager.SLOW_PERIODIC_JOBS.iteritems(): config.cron_worker.add_task(job, interval(i), wrap_slow(f)) def _unlocked_get_all_workers(config): return (config.mail_sources.values() + config.other_workers + [config.http_worker, config.tor_worker, config.slow_worker, config.scan_worker, config.cron_worker]) def stop_workers(config): try: self.index_check.release() except: pass with config._lock: worker_list = config._unlocked_get_all_workers() config.other_workers = [] config.tor_worker = None config.http_worker = None config.cron_worker = None config.slow_worker = config.dumb_worker config.scan_worker = config.dumb_worker for wait in (False, True): for w in worker_list: if w and w.isAlive(): if config.sys.debug and wait: print('Waiting for %s' % w) w.quit(join=wait) # Flush the mailbox cache (queues save worker jobs) config.flush_mbox_cache(config.background, clear=True) # Handle the save worker last, once all the others are # no longer feeding it new things to do. with config._lock: save_worker = config.save_worker config.save_worker = config.dumb_worker if config.sys.debug: print('Waiting for %s' % save_worker) from mailpile.postinglist import PLC_CACHE_FlushAndClean PLC_CACHE_FlushAndClean(config.background, keep=0) config.search_history.save(config) save_worker.quit(join=True) if config.sys.debug: # Hooray! print('All stopped!') def _unlocked_notify_workers_config_changed(config): worker_list = config._unlocked_get_all_workers() for worker in worker_list: if hasattr(worker, 'notify_config_changed'): worker.notify_config_changed() ############################################################################## if __name__ == "__main__": import copy import doctest import sys import mailpile.config.base import mailpile.config.defaults import mailpile.config.manager import mailpile.plugins.tags import mailpile.ui rules = copy.deepcopy(mailpile.config.defaults.CONFIG_RULES) rules.update({ 'nest1': ['Nest1', { 'nest2': ['Nest2', str, []], 'nest3': ['Nest3', { 'nest4': ['Nest4', str, []] }, []], }, {}] }) cfg = mailpile.config.manager.ConfigManager(rules=rules) session = mailpile.ui.Session(cfg) session.ui = mailpile.ui.SilentInteraction(cfg) session.ui.block() for tries in (1, 2): # This tests that we can set (and reset) dicts of unnested objects cfg.tags = {} assert(cfg.tags.a is None) for tn in range(0, 11): cfg.tags.append({'name': 'Test Tag %s' % tn}) assert(cfg.tags.a['name'] == 'Test Tag 10') # This tests the same thing for lists #cfg.profiles = [] #assert(len(cfg.profiles) == 0) #cfg.profiles.append({'name': 'Test Profile'}) #assert(len(cfg.profiles) == 1) #assert(cfg.profiles[0].name == 'Test Profile') # This is the complicated one: multiple nesting layers cfg.nest1 = {} assert(cfg.nest1.a is None) cfg.nest1.a = { 'nest2': ['hello', 'world'], 'nest3': [{'nest4': ['Hooray']}] } cfg.nest1.b = { 'nest2': ['hello', 'world'], 'nest3': [{'nest4': ['Hooray', 'Bravo']}] } assert(cfg.nest1.a.nest3[0].nest4[0] == 'Hooray') assert(cfg.nest1.b.nest3[0].nest4[1] == 'Bravo') assert(cfg.sys.http_port == mailpile.config.defaults.CONFIG_RULES['sys'][-1]['http_port'][-1]) assert(cfg.sys.path.vcards == 'vcards') assert(cfg.walk('sys.path.vcards') == 'vcards') # Verify that the tricky nested stuff from above persists and # load/save doesn't change lists. for passes in (1, 2, 3): cfg2 = mailpile.config.manager.ConfigManager(rules=rules) cfg2.parse_config(session, cfg.as_config_bytes()) cfg.parse_config(session, cfg2.as_config_bytes()) assert(cfg2.nest1.a.nest3[0].nest4[0] == 'Hooray') assert(cfg2.nest1.b.nest3[0].nest4[1] == 'Bravo') assert(len(cfg2.nest1) == 2) assert(len(cfg.nest1) == 2) assert(len(cfg.tags) == 11) results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={'cfg': cfg, 'session': session}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/config/paths.py ================================================ import os import sys import mailpile.platforms try: from appdirs import AppDirs except ImportError: AppDirs = None def _ensure_exists(path, mode=0o700): if not os.path.exists(path): head, tail = os.path.split(path) _ensure_exists(head) os.mkdir(path, mode) return path def LEGACY_DEFAULT_WORKDIR(profile): if profile == 'default': # Backwards compatibility: If the old ~/.mailpile exists, use it. workdir = os.path.expanduser('~/.mailpile') if os.path.exists(workdir) and os.path.isdir(workdir): return workdir return os.path.join( mailpile.platforms.GetAppDataDirectory(), 'Mailpile', profile) def DEFAULT_WORKDIR(): # The Mailpile environment variable trumps everything workdir = os.getenv('MAILPILE_HOME') if workdir: return _ensure_exists(workdir) # Which profile? profile = os.getenv('MAILPILE_PROFILE', 'default') # Check if we have a legacy setup we need to preserve workdir = LEGACY_DEFAULT_WORKDIR(profile) if not AppDirs or (os.path.exists(workdir) and os.path.isdir(workdir)): return _ensure_exists(workdir) # Use platform-specific defaults # via https://github.com/ActiveState/appdirs dirs = AppDirs("Mailpile", "Mailpile ehf") return _ensure_exists(os.path.join(dirs.user_data_dir, profile)) def DEFAULT_SHARED_DATADIR(): # IMPORTANT: This code is duplicated in mailpile-admin.py. # If it needs changing please change both places! env_share = os.getenv('MAILPILE_SHARED') if env_share is not None: return env_share # Check if we are running in a virtual env # http://stackoverflow.com/questions/1871549/python-determine-if-running-inside-virtualenv # We must also check that we are installed in the virtual env, # not just that we are running in a virtual env. if ((hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix')) and __file__.startswith(sys.prefix)): return os.path.join(sys.prefix, 'share', 'mailpile') # Check if we've been installed to /usr/local (or equivalent) usr_local = os.path.join(sys.prefix, 'local') if __file__.startswith(usr_local): return os.path.join(usr_local, 'share', 'mailpile') # Check if we are in /usr/ (sys.prefix) if __file__.startswith(sys.prefix): return os.path.join(sys.prefix, 'share', 'mailpile') # Else assume dev mode, source tree layout return os.path.join( os.path.dirname(__file__), '..', '..', 'shared-data') def DEFAULT_LOCALE_DIRECTORY(): """Get the gettext translation object, no matter where our CWD is""" return os.path.join(DEFAULT_SHARED_DATADIR(), "locale") def LOCK_PATHS(workdir=None): if workdir is None: workdir = DEFAULT_WORKDIR() return ( os.path.join(workdir, 'public-lock'), os.path.join(workdir, 'workdir-lock')) ================================================ FILE: mailpile/config/validators.py ================================================ from __future__ import print_function import os import socket import re try: import win_inet_pton except ImportError: pass from urlparse import urlparse from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as n from mailpile.util import * def BoolCheck(value): """ Convert common yes/no strings into boolean values. >>> BoolCheck('yes') True >>> BoolCheck('no') False >>> BoolCheck('true') True >>> BoolCheck('false') False >>> BoolCheck('on') True >>> BoolCheck('off') False >>> BoolCheck('wiggle') Traceback (most recent call last): ... ValueError: Invalid boolean: wiggle """ bool_val = truthy(value, default=None) if bool_val is None: raise ValueError(_('Invalid boolean: %s') % value) return bool_val def SlugCheck(slug, allow=''): """ Verify that a string is a valid URL slug. >>> SlugCheck('_Foo-bar.5') '_foo-bar.5' >>> SlugCheck('Bad Slug') Traceback (most recent call last): ... ValueError: Invalid URL slug: Bad Slug >>> SlugCheck('Bad/Slug') Traceback (most recent call last): ... ValueError: Invalid URL slug: Bad/Slug """ if not slug == CleanText(unicode(slug), banned=(CleanText.NONDNS.replace(allow, '')) ).clean: raise ValueError(_('Invalid URL slug: %s') % slug) return slug.lower() def SlashSlugCheck(slug): """ Verify that a string is a valid URL slug (slashes allowed). >>> SlashSlugCheck('Okay/Slug') 'okay/slug' """ return SlugCheck(slug, allow='/') def RouteProtocolCheck(proto): """ Verify that the protocol is actually a protocol. (FIXME: Should reference a list of registered protocols...) >>> RouteProtocolCheck('SMTP') 'smtp' """ proto = str(proto).strip().lower() if proto not in ("smtp", "smtptls", "smtpssl", "local"): raise ValueError(_('Invalid message delivery protocol: %s') % proto) return proto def DnsNameValid(dnsname): """ Tests whether a string is a valid dns name, returns a boolean value """ if not dnsname or not DNSNAME_RE.match(dnsname): return False else: return True def HostNameValid(host): """ Tests whether a string is a valid host-name, return a boolean value >>> HostNameValid("127.0.0.1") True >>> HostNameValid("::1") True >>> HostNameValid("localhost") True >>> HostNameValid("22.45") False """ valid = False for attr in ["AF_INET","AF_INET6"]: try: socket.inet_pton(socket.__getattribute__(attr), host) valid = True break except (socket.error): pass if not valid: # the host is not an IP so check if its a hostname i.e. 'localhost' or 'site.com' if not host or (not DnsNameValid(host) and not ALPHA_RE.match(host)): return False else: return True else: return True def HostNameCheck(host): """ Verify that a string is a valid host-name, return it lowercased. >>> HostNameCheck('foo.BAR.baz') 'foo.bar.baz' >>> HostNameCheck('127.0.0.1') '127.0.0.1' >>> HostNameCheck('not/a/hostname') Traceback (most recent call last): ... ValueError: Invalid hostname: not/a/hostname """ # Check DNS, IPv4, and finally IPv6 if not HostNameValid(host): raise ValueError(_('Invalid hostname: %s') % host) return str(host).lower() def B36Check(b36val): """ Verify that a string is a valid path base-36 integer. >>> B36Check('Aa') 'aa' >>> B36Check('.') Traceback (most recent call last): ... ValueError: invalid ... """ int(b36val, 36) return str(b36val).lower() def NotUnicode(string): """ Make sure a string is NOT unicode. """ if isinstance(string, unicode): string = string.encode('utf-8') if not isinstance(string, str): return str(string) return string def PathCheck(path): """ Verify that a string is a valid path, make it absolute. >>> PathCheck('/etc/../') '/' >>> PathCheck('/no/such/path') Traceback (most recent call last): ... ValueError: File/directory does not exist: /no/such/path """ if isinstance(path, unicode): path = path.encode('utf-8') path = os.path.expanduser(path) if not os.path.exists(path): raise ValueError(_('File/directory does not exist: %s') % path) return os.path.abspath(path) def WebRootCheck(path): """ Verify that a string is a valid web root path, normalize the slashes. >>> WebRootCheck('/') '' >>> WebRootCheck('/foo//bar////baz//') '/foo/bar/baz' >>> WebRootCheck('/foo/$%!') Traceback (most recent call last): ... ValueError: Invalid web root: /foo/$%! """ p = re.sub('/+', '/', '/%s/' % path)[:-1] if (p != CleanText(p, banned=CleanText.NONPATH).clean): raise ValueError('Invalid web root: %s' % path) return p def FileCheck(path=None): """ Verify that a string is a valid path to a file, make it absolute. >>> FileCheck('/etc/../etc/passwd') '/etc/passwd' >>> FileCheck('/') Traceback (most recent call last): ... ValueError: Not a file: / """ if path in (None, 'None', 'none', ''): return None path = PathCheck(path) if not os.path.isfile(path): raise ValueError(_('Not a file: %s') % path) return path def DirCheck(path=None): """ Verify that a string is a valid path to a directory, make it absolute. >>> DirCheck('/etc/../') '/' >>> DirCheck('/etc/passwd') Traceback (most recent call last): ... ValueError: Not a directory: /etc/passwd """ if path in (None, 'None', 'none', ''): return None path = PathCheck(path) if not os.path.isdir(path): raise ValueError(_('Not a directory: %s') % path) return path def NewPathCheck(path): """ Verify that a string is a valid path to a directory, make it absolute. >>> NewPathCheck('/magic') '/magic' >>> NewPathCheck('/no/such/path/magic') Traceback (most recent call last): ... ValueError: File/directory does not exist: /no/such/path """ PathCheck(os.path.dirname(path)) return os.path.abspath(path) def UrlCheck(url): """ Verify that a url parsed string has a valid uri scheme >>> UrlCheck("http://mysite.com") 'http://mysite.com' >>> UrlCheck("/not-valid.net") Traceback (most recent call last): ... ValueError: Not a valid url: ... >>> UrlCheck("tallnet://some-host.com") Traceback (most recent call last): ... ValueError: Not a valid url: tallnet://some-host.com """ uri = urlparse(url) if not uri.scheme in URI_SCHEMES: raise ValueError(_("Not a valid url: %s") % url) else: return url def EmailCheck(email): """ Verify that a string is a valid email >>> EmailCheck("test@test.com") 'test@test.com' """ if not EMAIL_RE.match(email): raise ValueError(_("Not a valid e-mail: %s") % email) return email def GPGKeyCheck(value): """ Strip a GPG fingerprint of all spaces, make sure it seems valid. Will also accept e-mail addresses, for legacy reasons. >>> GPGKeyCheck('User@Foo.com') 'User@Foo.com' >>> GPGKeyCheck('1234 5678 abcd EF00') '12345678ABCDEF00' >>> GPGKeyCheck('12345678') '12345678' >>> GPGKeyCheck('B906 EA4B 8A28 15C4 F859 6F9F 47C1 3F3F ED73 5179') 'B906EA4B8A2815C4F8596F9F47C13F3FED735179' >>> GPGKeyCheck('B906 8A28 15C4 F859 6F9F 47C1 3F3F ED73 5179') Traceback (most recent call last): ... ValueError: Not a GPG key ID or fingerprint >>> GPGKeyCheck('B906 8X28 1111 15C4 F859 6F9F 47C1 3F3F ED73 5179') Traceback (most recent call last): ... ValueError: Not a GPG key ID or fingerprint """ value = value.replace(' ', '').replace('\t', '').strip() if value in ('!CREATE', '!PASSWORD'): return value try: if len(value) not in (8, 16, 40): raise ValueError(_('Not a GPG key ID or fingerprint')) if re.match(r'^[0-9A-F]+$', value.upper()) is None: raise ValueError(_('Not a GPG key ID or fingerprint')) except ValueError: try: return EmailCheck(value) except ValueError: raise ValueError(_('Not a GPG key ID or fingerprint')) return value.upper() class IgnoreValue(Exception): pass def IgnoreCheck(data): raise IgnoreValue() if __name__ == "__main__": import doctest import sys result = doctest.testmod(optionflags=doctest.ELLIPSIS) print('%s' % (result, )) if result.failed: sys.exit(1) ================================================ FILE: mailpile/conn_brokers.py ================================================ from __future__ import print_function # Connection brokers facilitate & manage incoming and outgoing connections. # # The idea is that code actually tells us what it wants to do, so we can # choose an appropriate mechanism for connecting or receiving incoming # connections. # # Libraries which use socket.create_connection can be monkey-patched # to use a broker on a connection-by-connection bases like so: # # with broker.context(need=[broker.OUTGOING_CLEARTEXT, # broker.OUTGOING_SMTP]) as ctx: # conn = somelib.connect(something) # print 'Connected with encryption: %s' % ctx.encryption # # The context variable will then contain metadata about what sort of # connection was made. # # See the Capability class below for a list of attributes that can be # used to describe an outgoing (or incoming) connection. # # In particular, using the master broker will implement a prioritised # connection strategy where the most secure options are tried first and # things gracefully degrade. Protocols like IMAP, SMTP or POP3 will be # transparently upgraded to use STARTTLS. # # TODO: # - Implement a TorBroker # - Implement a PageKiteBroker # - Implement HTTP/SMTP/IMAP/POP3 TLS upgrade-brokers # - Prevent unbrokered socket.socket connections # import datetime import socket import ssl import subprocess import sys import threading import time import traceback try: import cryptography import cryptography.hazmat.backends import cryptography.hazmat.primitives.hashes try: import cryptography.x509 as cryptography_x509 except ImportError: cryptography_x509 = None except ImportError: cryptography = None # Import SOCKS proxy support... try: import sockschain as socks except ImportError: try: import socks except ImportError: socks = None import mailpile.security as security from mailpile.i18n import gettext from mailpile.i18n import ngettext as _n from mailpile.commands import Command from mailpile.util import md5_hex, dict_merge, monkey_patch from mailpile.security import tls_sock_cert_sha256 _ = lambda s: s KNOWN_ONION_MAP = { 'www.mailpile.is': 'clgs64523yi2bkhz.onion' } original_scc = socket.create_connection monkey_clean_scc = socket.create_connection monkey_thread_local = threading.local() def MonkeySockCreateConn(*args, **kwargs): if hasattr(monkey_thread_local, 'scc'): cconn = (monkey_thread_local.scc or [monkey_clean_scc])[-1] else: cconn = monkey_clean_scc return cconn(*args, **kwargs) def _explain_encryption(sock): try: algo, proto, bits = sock.cipher() return ( _('%(tls_version)s (%(bits)s bit %(algorithm)s)') ) % { 'bits': bits, 'tls_version': proto, 'algorithm': algo} except (ValueError, AttributeError): return _('no encryption') class Capability(object): """ These are constants defining different types of outgoing or incoming connections. Brokers use these to describe what sort of connections they are capable of handling, and calling code uses these to describe the intent of network connection. """ OUTGOING_RAW = 'o:raw' # Request this to avoid meddling brokers OUTGOING_ENCRYPTED = 'o:e' # Request this if sending encrypted data OUTGOING_CLEARTEXT = 'o:c' # Request this if sending clear-text data OUTGOING_TRACKABLE = 'o:t' # Reject this to require anonymity OUTGOING_SMTP = 'o:smtp' # These inform brokers what protocol is being OUTGOING_IMAP = 'o:imap' # .. used, to allow protocol-specific features OUTGOING_POP3 = 'o:pop3' # .. such as enabling STARTTLS or upgrading OUTGOING_HTTP = 'o:http' # .. HTTP to HTTPS. OUTGOING_HTTPS = 'o:https' # .. OUTGOING_SMTPS = 'o:smtps' # .. OUTGOING_POP3S = 'o:pop3s' # .. OUTGOING_IMAPS = 'o:imaps' # .. INCOMING_RAW = 20 INCOMING_LOCALNET = 21 INCOMING_INTERNET = 22 INCOMING_DARKNET = 23 INCOMING_SMTP = 24 INCOMING_IMAP = 25 INCOMING_POP3 = 26 INCOMING_HTTP = 27 INCOMING_HTTPS = 28 ALL_OUTGOING = set([OUTGOING_RAW, OUTGOING_ENCRYPTED, OUTGOING_CLEARTEXT, OUTGOING_TRACKABLE, OUTGOING_SMTP, OUTGOING_IMAP, OUTGOING_POP3, OUTGOING_SMTPS, OUTGOING_IMAPS, OUTGOING_POP3S, OUTGOING_HTTP, OUTGOING_HTTPS]) ALL_OUTGOING_ENCRYPTED = set([OUTGOING_RAW, OUTGOING_TRACKABLE, OUTGOING_ENCRYPTED, OUTGOING_HTTPS, OUTGOING_SMTPS, OUTGOING_POP3S, OUTGOING_IMAPS]) ALL_INCOMING = set([INCOMING_RAW, INCOMING_LOCALNET, INCOMING_INTERNET, INCOMING_DARKNET, INCOMING_SMTP, INCOMING_IMAP, INCOMING_POP3, INCOMING_HTTP, INCOMING_HTTPS]) class CapabilityFailure(IOError): """ This exception is raised when capability requirements can't be satisfied. It extends the IOError, so unaware code just thinks the network is lame. >>> try: ... raise CapabilityFailure('boo') ... except IOError: ... print('ok') ok """ pass class Url(str): def __init__(self, *args, **kwargs): str.__init__(self, *args, **kwargs) self.encryption = None self.anonymity = None self.on_internet = False self.on_localnet = False self.on_darknet = None class BrokeredContext(object): """ This is the context returned by the BaseConnectionBroker.context() method. It takes care of monkey-patching the socket.create_connection method and then cleaning the mess up afterwards, and collecting metadata from the brokers describing what sort of connection was established. WARNING: In spite of our best efforts (locking, etc.), mixing brokered and unbrokered code will not work well at all. The patching approach also limits us to initiating one outgoing connection at a time. """ def __init__(self, broker, need=None, reject=None, oneshot=False): self._broker = broker self._need = need self._reject = reject self._oneshot = oneshot self._reset() def __str__(self): hostport = '%s:%s' % (self.address or ('unknown', 'none')) if self.error: return _('Failed to connect to %s: %s') % (hostport, self.error) if self.anonymity: network = self.anonymity elif self.on_darknet: network = self.on_darknet elif self.on_localnet: network = _('the local network') elif self.on_internet: network = _('the Internet') else: return _('Attempting to connect to %(host)s') % {'host': hostport} return _('Connected to %(host)s over %(network)s with %(encryption)s.' ) % { 'network': network, 'host': hostport, 'encryption': self.encryption or _('no encryption') } def _reset(self): self.error = None self.address = None self.encryption = None self.anonymity = None self.on_internet = False self.on_localnet = False self.on_darknet = None def __enter__(self, *args, **kwargs): def create_brokered_conn(address, *a, **kw): self._reset() return self._broker.create_conn_with_caps( address, self, self._need, self._reject, *a, **kw) if hasattr(monkey_thread_local, 'scc'): monkey_thread_local.scc.append(create_brokered_conn) else: monkey_thread_local.scc = [create_brokered_conn] if socket.create_connection != MonkeySockCreateConn: socket.create_connection = MonkeySockCreateConn return self def __exit__(self, *args, **kwargs): monkey_thread_local.scc.pop(-1) class BaseConnectionBroker(Capability): """ This is common code used by most of the connection brokers. """ SUPPORTS = [] def __init__(self, master=None): self.supports = list(self.SUPPORTS)[:] self.master = master self._config = None self._debug = master._debug if (master is not None) else None def configure(self): self.supports = list(self.SUPPORTS)[:] def set_config(self, config): self._config = config self.configure() def config(self): if self._config is not None: return self._config if self.master is not None: return self.master.config() return None def _raise_or_none(self, exc, why): if exc is not None: raise exc(why) return None def _check(self, need, reject, _raise=CapabilityFailure): for n in need or []: if n not in self.supports: if self._debug is not None: self._debug('%s: lacking capabilty %s' % (self, n)) return self._raise_or_none(_raise, 'Lacking %s' % n) for n in reject or []: if n in self.supports: if self._debug is not None: self._debug('%s: unwanted capabilty %s' % (self, n)) return self._raise_or_none(_raise, 'Unwanted %s' % n) if self._debug is not None: self._debug('%s: checks passed!' % (self, )) return self def _describe(self, context, conn): return conn def debug(self, val): self._debug = val return self def context(self, need=None, reject=None, oneshot=False): return BrokeredContext(self, need=need, reject=reject, oneshot=oneshot) def create_conn_with_caps(self, address, context, need, reject, *args, **kwargs): if context.address is None: context.address = address conn = self._check(need, reject)._create_connection(context, address, *args, **kwargs) return self._describe(context, conn) def create_connection(self, address, *args, **kwargs): n = kwargs.get('need', None) r = kwargs.get('reject', None) c = kwargs.get('context', None) for kw in ('need', 'reject', 'context'): if kw in kwargs: del kwargs[kw] return self.create_conn_with_caps(address, c, n, r, *args, **kwargs) # Should implement socket.create_connection or an equivalent. # Context, if not None, should be informed with metadata about the # connection. def _create_connection(self, context, address, *args, **kwargs): raise NotImplementedError('Subclasses override this') def get_urls(self, listening_fd, need=None, reject=None, **kwargs): try: return self._check(need, reject)._get_urls(listening_fd, **kwargs) except CapabilityFailure: return [] # Returns a list of Url objects for this listener def _get_urls(self, listening_fd, proto=None, username=None, password=None): raise NotImplementedError('Subclasses override this') class TcpConnectionBroker(BaseConnectionBroker): """ The basic raw TCP/IP connection broker. The only clever thing this class does, is to avoid trying to connect to .onion addresses, preventing that from leaking over DNS. """ SUPPORTS = ( # Normal TCP/IP is not anonymous, and we do not have incoming # capability unless we have a public IP. (Capability.ALL_OUTGOING) | (Capability.ALL_INCOMING - set([Capability.INCOMING_INTERNET])) ) LOCAL_NETWORKS = ['localhost', '127.0.0.1', '::1'] FIXED_NO_PROXY_LIST = ['localhost', '127.0.0.1', '::1'] DEBUG_FMT = '%s: Raw TCP conn to: %s' def configure(self): BaseConnectionBroker.configure(self) # FIXME: If our config indicates we have a public IP, add the # INCOMING_INTERNET capability. def _describe(self, context, conn): try: (host, port) = conn.getpeername()[:2] if host.lower() in self.LOCAL_NETWORKS: context.on_localnet = True else: context.on_internet = True except TypeError: # conn.getpeername() may return None pass context.encryption = None return conn def _in_no_proxy_list(self, address): cfg_no_proxy = self.config().get_proxy_settings().get('no_proxy', '') no_proxy = (self.FIXED_NO_PROXY_LIST + [a.lower().strip() for a in cfg_no_proxy.split(',')]) return (address[0].lower() in no_proxy) def _avoid(self, address): proxy_settings = self.config().get_proxy_settings() if (proxy_settings['protocol'] not in ('none', 'unknown', 'system') and not proxy_settings.get('fallback', False) and not self._in_no_proxy_list(address)): raise CapabilityFailure('Proxy fallback is disabled') def _broker_avoid(self, address): if address[0].endswith('.onion'): raise CapabilityFailure('Cannot connect to .onion addresses') def _conn(self, address, *args, **kwargs): clean_kwargs = dict((k, v) for k, v in kwargs.iteritems() if not k.startswith('_')) return original_scc(address, *args, **clean_kwargs) def _create_connection(self, context, address, *args, **kwargs): self._avoid(address) self._broker_avoid(address) if self._debug is not None: self._debug(self.DEBUG_FMT % (self, address)) return self._conn(address, *args, **kwargs) class SocksConnBroker(TcpConnectionBroker): """ This broker offers the same services as the TcpConnBroker, but over a SOCKS connection. """ SUPPORTS = [] CONFIGURED = Capability.ALL_OUTGOING PROXY_TYPES = ('socks5', 'http', 'socks4') DEFAULT_PROTO = 'socks5' DEBUG_FMT = '%s: Raw SOCKS5 conn to: %s' IOERROR_FMT = _('SOCKS error, %s') IOERROR_MSG = { 'timed out': _('timed out'), 'Host unreachable': _('host unreachable'), 'Connection refused': _('connection refused') } def _describe(self, context, conn): context.on_darknet = ('proxy (%s:%d)' % (self.proxy_config['host'], self.proxy_config['port'])) return conn def __init__(self, *args, **kwargs): TcpConnectionBroker.__init__(self, *args, **kwargs) self.proxy_config = None self.typemap = {} def configure(self): BaseConnectionBroker.configure(self) proxy_settings = self.config().get_proxy_settings() if proxy_settings.get('protocol') in self.PROXY_TYPES: self.proxy_config = proxy_settings self.supports = list(self.CONFIGURED)[:] self.typemap = { 'socks5': socks.PROXY_TYPE_SOCKS5, 'socks4': socks.PROXY_TYPE_SOCKS4, 'tor': socks.PROXY_TYPE_SOCKS5, # For TorConnBroker 'tor-risky': socks.PROXY_TYPE_SOCKS5, # For TorConnBroker 'http': socks.PROXY_TYPE_HTTP} else: self.proxy_config = None self.supports = [] def _auth_args(self): return { 'username': self.proxy_config.get('username') or None, 'password': self.proxy_config.get('password') or None} def _avoid(self, address): if self._in_no_proxy_list(address): raise CapabilityFailure('Proxy to %s:%s disabled by policy' ) % address def _fix_address_tuple(self, address): return (str(address[0]), int(address[1])) def _conn(self, address, timeout=None, source_address=None, **kwargs): sock = socks.socksocket() proxytype = self.typemap.get(self.proxy_config.get('protocol'), self.typemap[self.DEFAULT_PROTO]) sock.setproxy(proxytype=proxytype, addr=self.proxy_config.get('host'), port=int(self.proxy_config.get('port', 0)), rdns=True, **self._auth_args()) if timeout and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(float(timeout)) if source_address: raise CapabilityFailure('Cannot bind source address') try: address = self._fix_address_tuple(address) sock.connect(address) except socks.ProxyError as e: if self._debug is not None: self._debug(traceback.format_exc()) code, msg = e.message raise IOError(_(self.IOERROR_FMT ) % (_(self.IOERROR_MSG.get(msg, msg)), )) return sock class TorConnBroker(SocksConnBroker): """ This broker offers the same services as the TcpConnBroker, but over Tor. This removes the "trackable" capability, so requests that reject it can find their way here safely... This broker only volunteers to carry encrypted traffic, because Tor exit nodes may be hostile. """ SUPPORTS = [] CONFIGURED = (Capability.ALL_OUTGOING_ENCRYPTED - set([Capability.OUTGOING_TRACKABLE])) REJECTS = None PROXY_TYPES = ('tor', ) DEFAULT_PROTO = 'tor' DEBUG_FMT = '%s: Raw Tor conn to: %s' IOERROR_FMT = _('Tor error, %s') IOERROR_MSG = dict_merge(SocksConnBroker.IOERROR_MSG, { 'bad input': _('connection refused') # FIXME: Is this right? }) def _describe(self, context, conn): context.on_darknet = 'Tor' context.anonymity = 'Tor' return conn def _auth_args(self): # FIXME: Tor uses the auth information as a signal to change # circuits. We may have use for this at some point. return {} def _fix_address_tuple(self, address): host = str(address[0]) return (KNOWN_ONION_MAP.get(host.lower(), host), int(address[1])) def _broker_avoid(self, address): # Disable the avoiding of .onion addresses added above pass class TorRiskyBroker(TorConnBroker): """ This differs from the TorConnBroker in that it will allow "cleartext" traffic to anywhere - this is dangerous, because exit nodes could mess with our traffic. """ CONFIGURED = (Capability.ALL_OUTGOING - set([Capability.OUTGOING_TRACKABLE])) DEBUG_FMT = '%s: Risky Tor conn to: %s' PROXY_TYPES = ('tor-risky', ) DEFAULT_PROTO = 'tor-risky' class TorOnionBroker(TorConnBroker): """ This broker offers the same services as the TcpConnBroker, but over Tor. This removes the "trackable" capability, so requests that reject it can find their way here safely... This differs from the TorConnBroker in that it will allow "cleartext" traffic, since we trust the traffic never leaves the Tor network and we don't have hostile exits to worry about. """ SUPPORTS = [] CONFIGURED = (Capability.ALL_OUTGOING - set([Capability.OUTGOING_TRACKABLE])) REJECTS = None DEBUG_FMT = '%s: Tor onion conn to: %s' PROXY_TYPES = ('tor', 'tor-risky') def _broker_avoid(self, address): host = KNOWN_ONION_MAP.get(address[0], address[0]) if not host.endswith('.onion'): raise CapabilityFailure('Can only connect to .onion addresses') class BaseConnectionBrokerProxy(TcpConnectionBroker): """ Brokers based on this establish a RAW connection and then manipulate it in some way, generally to implement proxying or TLS wrapping. """ SUPPORTS = [] WANTS = [Capability.OUTGOING_RAW] REJECTS = None def _proxy_address(self, address): return address def _proxy(self, conn): raise NotImplementedError('Subclasses override this') def _wrap_ssl(self, conn): if self._debug is not None: self._debug('%s: Wrapping socket with SSL' % (self, )) return ssl.wrap_socket(conn) def _create_connection(self, context, address, *args, **kwargs): address = self._proxy_address(address) if self.master: conn = self.master.create_conn_with_caps( address, context, self.WANTS, self.REJECTS, *args, **kwargs) else: conn = TcpConnectionBroker._create_connection(self, context, address, *args, **kwargs) return self._proxy(conn) class AutoTlsConnBroker(BaseConnectionBrokerProxy): """ This broker tries to auto-upgrade connections to use TLS, or at least do the SSL handshake here so we can record info about it. """ SUPPORTS = [Capability.OUTGOING_HTTP, Capability.OUTGOING_HTTPS, Capability.OUTGOING_IMAPS, Capability.OUTGOING_SMTPS, Capability.OUTGOING_POP3S] WANTS = [Capability.OUTGOING_RAW, Capability.OUTGOING_ENCRYPTED] def _describe(self, context, conn): context.encryption = _explain_encryption(conn) return conn def _proxy_address(self, address): if address[0].endswith('.onion'): raise CapabilityFailure('I do not like .onion addresses') if int(address[1]) != 443: # FIXME: Import HTTPS Everywhere database to make this work? raise CapabilityFailure('Not breaking clear-text HTTP yet') return address def _proxy(self, conn): return self._wrap_ssl(conn) class AutoSmtpStartTLSConnBroker(BaseConnectionBrokerProxy): pass class AutoImapStartTLSConnBroker(BaseConnectionBrokerProxy): pass class AutoPop3StartTLSConnBroker(BaseConnectionBrokerProxy): pass class MasterBroker(BaseConnectionBroker): """ This is the master broker. It implements a prioritised list of connection brokers, each of which is tried in turn until a match is found. As such, more secure brokers should register themselves with a higher priority - if they fail, we fall back to less secure connection strategies. """ def __init__(self, *args, **kwargs): BaseConnectionBroker.__init__(self, *args, **kwargs) self.brokers = [] self.history = [] self._debug = self._debugger self.debug_callback = None def configure(self): for prio, cb in self.brokers: cb.configure() def _debugger(self, *args, **kwargs): if self.debug_callback is not None: self.debug_callback(*args, **kwargs) def register_broker(self, priority, cb): """ Brokers should register themselves with priorities as follows: - 1000-1999: Content-agnostic raw connections - 3000-3999: Secure network layers: VPNs, Tor, I2P, ... - 5000-5999: Proxies required to reach the wider Internet - 7000-7999: Protocol enhancments (non-security related) - 9000-9999: Security-related protocol enhancements """ self.brokers.append((priority, cb(master=self))) self.brokers.sort() self.brokers.reverse() def get_fd_context(self, fileno): for t, fd, context in reversed(self.history): if fd == fileno: return context return BrokeredContext(self) def create_conn_with_caps(self, address, context, need, reject, *args, **kwargs): history_event = kwargs.get('_history_event') if history_event is None: history_event = [int(time.time()), None, context] self.history = self.history[-50:] self.history.append(history_event) kwargs['_history_event'] = history_event else: history_event[-1] = context if context.address is None: context.address = address et = v = t = None for prio, cb in self.brokers: try: conn = cb.debug(self._debug).create_conn_with_caps( address, context, need, reject, *args, **kwargs) if conn: history_event[1] = conn.fileno() return conn except (CapabilityFailure, NotImplementedError): # These are internal; we assume they're already logged # for debugging but don't bother the user with them. pass except: et, v, t = sys.exc_info() if et is not None: context.error = '%s' % v raise et, v, t context.error = _('No connection method found') raise CapabilityFailure(context.error) def get_urls(self, listening_fd, need=None, reject=None): urls = [] for prio, cb in self.brokers: urls.extend(cb.debug(self._debug).get_urls(listening_fd)) return urls def DisableUnbrokeredConnections(): """Enforce the use of brokers EVERYWHERE!""" def CreateConnWarning(*args, **kwargs): print('*** socket.create_connection used without a broker ***') traceback.print_stack() raise IOError('FIXME: Please use within a broker context') monkey_clean_scc = CreateConnWarning socket.create_connection = CreateConnWarning class NetworkHistory(Command): """Show recent network history""" SYNOPSIS = (None, 'logs/network', 'logs/network', None) ORDER = ('Internals', 6) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False class CommandResult(Command.CommandResult): def as_text(self): if self.result: def fmt(result): dt = datetime.datetime.fromtimestamp(result[0]) return '%2.2d:%2.2d %s' % (dt.hour, dt.minute, result[-1]) return '\n'.join(fmt(r) for r in self.result) return _('No network events recorded') def command(self): return self._success(_('Listed recent network events'), result=Master.history) class GetTlsCertificate(Command): """Fetch and parse a server's TLS certificate""" SYNOPSIS = (None, 'crypto/tls/getcert', 'crypto/tls/getcert', '[--tofu-save|--tofu-clear]') ORDER = ('Internals', 6) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = { 'tofu-clear': 'Remove from TOFU certificate store', 'tofu-save': 'Save to our TOFU certificate store', 'host': 'Name of remote server' } class CommandResult(Command.CommandResult): def as_text(self): if self.result: def fmt(h, r): return '%s:\t%s' % (h, r[-1] or r[1]) return '\n'.join(fmt(h, r) for h, r in self.result.iteritems()) return _('No certificates found') def command(self): if self.data.get('_method', 'POST') != 'POST': # Allow HTTP GET as a no-op, so the user can see a friendly form. return self._success(_('Examine TLS certificates')) config = self.session.config tofu_save = self.data.get('tofu-save', '--tofu-save' in self.args) tofu_clear = self.data.get('tofu-clear', '--tofu-clear' in self.args) hosts = (list(s for s in self.args if not s.startswith('--')) + self.data.get('host', [])) def ts(t): return int(time.mktime(t.timetuple())) def oidName(oid): return { '2.5.4.3': 'commonName', '2.5.4.4': 'surname', '2.5.4.5': 'serialNumber', '2.5.4.6': 'countryName', '2.5.4.7': 'localityName', '2.5.4.8': 'stateOrProvinceName', '2.5.4.9': 'streetAddress', '2.5.4.10': 'organizationName', '2.5.4.11': 'organizationalUnitName' }.get(oid.dotted_string, getattr(oid, '_name', oid.dotted_string)) def oidmap(entries): return dict((oidName(e.oid), e.value) for e in entries) def subjmap(stext): def subjpair(kv): k, v = kv.split('=', 1) return ({'CN': 'commonName', 'C': 'countryName', 'ST': 'stateOrProvinceName', 'L': 'localityName', 'O': 'organizationName', 'OU': 'organizationalUnitName'}.get(k, k), v) parts = [] for part in stext.strip().split('/'): if '=' in part: parts.append(part) elif parts: parts[-1] += '/' + part return dict(subjpair(kv) for kv in parts) def fingerprint(cert_sha_256): fp = ['%2.2x' % ord(b) for b in cert_sha_256] fp2 = [fp[i*2] + fp[i*2 + 1] for i in range(0, len(fp)/2)] return fp2 def pts(t): dt, tz = t.rsplit(' ', 1) # Strip off the timezone return datetime.datetime.strptime(dt, '%b %d %H:%M:%S %Y') def parse_pem_cert(cert_pem, s256): cert_sha_256 = s256.decode('base64') now = datetime.datetime.today() if cryptography_x509 is None: # Shell out to openssl, boo. (stdout, stderr) = subprocess.Popen( ['openssl', 'x509', '-subject', '-issuer', '-dates', '-noout'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE).communicate(input=str(cert_pem)) if not stdout: raise ValueError(stderr) details = dict(l.split('=', 1) for l in stdout.strip().splitlines() if l and '=' in l) details['notAfter'] = pts(details['notAfter']) details['notBefore'] = pts(details['notBefore']) return { 'fingerprint': fingerprint(cert_sha_256), 'date_matches': False, 'date_matches': ((details['notBefore'] < now) and (details['notAfter'] > now)), 'not_valid_before': ts(details['notBefore']), 'not_valid_after': ts(details['notAfter']), 'subject': subjmap(details['subject']), 'issuer': subjmap(details['issuer'])} else: parsed = cryptography_x509.load_pem_x509_certificate( str(cert_pem), cryptography.hazmat.backends.default_backend()) return { 'fingerprint': fingerprint(cert_sha_256), 'date_matches': ((parsed.not_valid_before < now) and (parsed.not_valid_after > now)), 'not_valid_before': ts(parsed.not_valid_before), 'not_valid_after': ts(parsed.not_valid_after), 'subject': oidmap(parsed.subject), 'issuer': oidmap(parsed.issuer)} def attempt_starttls(addr, sock): # Attempt a minimal SMTP interaction, for STARTTLS support # We attempt a non-blocking peek unless we're sure this is # a port normally used for clear-text SMTP. peeking = int(addr[1]) not in (25, 587, 143) # If this isn't a known TLS port, then we sleep a bit to give a # greeting time to arrive. if peeking and int(addr[1]) not in (443, 465, 993, 995): time.sleep(0.4) try: # Look for an SMTP (or IMAP) greeting if peeking: sock.setblocking(0) # Note: This will throw a TypeError if we are connected # over Tor (or other SOCKS). first = sock.recv(1024, socket.MSG_PEEK) or '' else: sock.settimeout(10) first = sock.recv(1024) or '' if first[:4] == '220 ': # This is an SMTP greeting if peeking: sock.setblocking(1) sock.recv(1024) sock.sendall('EHLO example.com\r\n') if (sock.recv(1024) or '')[:1] == '2': sock.sendall('STARTTLS\r\n') sock.recv(1024) elif first[:4] == '* OK': # This is an IMAP4 greeting if peeking: sock.setblocking(1) sock.recv(1024) sock.sendall('* STARTTLS\r\n') sock.recv(1024) except (TypeError, IOError, OSError): pass finally: sock.setblocking(1) certs = {} ok = changes = 0 for host in hosts: try: addr = host.replace(' ', '').split(':') + ['443'] addr = (addr[0], int(addr[1])) try: with Master.context(need=[Master.OUTGOING_ENCRYPTED, Master.OUTGOING_RAW]) as ctx: sock = socket.create_connection(addr, timeout=30) attempt_starttls(addr, sock) ssls = ssl.wrap_socket(sock, use_web_ca=True, tofu=False) hostname_matches = True cert_validated = True except (ssl.SSLError, ssl.CertificateError) as e: if isinstance(e, ssl.CertificateError): cert_validated = True hostname_matches = False else: cert_validated = False hostname_matches = 'unknown' with Master.context(need=[Master.OUTGOING_ENCRYPTED, Master.OUTGOING_RAW]) as ctx: sock = socket.create_connection(addr, timeout=30) attempt_starttls(addr, sock) ssls = ssl.wrap_socket(sock, use_web_ca=False, tofu=False) cert = ssls.getpeercert(True) s256 = tls_sock_cert_sha256(cert=cert) ssls.close() cfg_key = md5_hex('%s:%d' % addr) if tofu_clear: if cfg_key in config.tls.keys(): del config.tls[cfg_key] changes += 1 if tofu_save: if cfg_key not in config.tls.keys(): config.tls[cfg_key] = {'server': '%s:%d' % addr} cert_tofu = config.tls[cfg_key] cert_tofu.use_web_ca = False cert_tofu.accept_certs.append(s256) changes += 1 else: cert_tofu = config.tls.get(cfg_key, {}) tofu_seen = s256 in cert_tofu.get('accept_certs', []) using_tofu = not cert_tofu.get('use_web_ca', True) cert = { 'current_time': int(time.time()), 'cert_validated': cert_validated, 'hostname_matches': hostname_matches, 'tofu_seen': tofu_seen, 'using_tofu': using_tofu, 'tofu_invalid': (using_tofu and not tofu_seen), 'pem': ssl.DER_cert_to_PEM_cert(cert)} cert.update(parse_pem_cert(cert['pem'], s256)) certs[host] = (True, s256, cert, None) ok += 1 except Exception as e: certs[host] = ( False, _('Failed to fetch certificate'), unicode(e), traceback.format_exc()) if changes: self._background_save(config=True) if ok: return self._success(_('Downloaded TLS certificates'), result=certs) else: return self._error(_('Failed to download TLS certificates'), result=certs) def SslWrapOnlyOnce(org_sslwrap, sock, *args, **kwargs): """ Since we like to wrap things our own way, this make ssl.wrap_socket into a no-op in the cases where we've alredy wrapped a socket. """ if not isinstance(sock, ssl.SSLSocket): ctx = Master.get_fd_context(sock.fileno()) try: if 'server_hostname' not in kwargs: kwargs['server_hostname'] = ctx.address[0] sock = org_sslwrap(sock, *args, **kwargs) ctx.encryption = _explain_encryption(sock) except (socket.error, IOError, ssl.SSLError, ssl.CertificateError) as e: ctx.error = '%s' % e raise return sock def SslContextWrapOnlyOnce(org_ctxwrap, self, sock, *args, **kwargs): return SslWrapOnlyOnce( lambda s, *a, **kwa: org_ctxwrap(self, s, *a, **kwa), sock, *args, **kwargs) _ = gettext if __name__ != "__main__": Master = MasterBroker() register = Master.register_broker register(1000, TcpConnectionBroker) register(9500, AutoTlsConnBroker) register(9500, AutoSmtpStartTLSConnBroker) register(9500, AutoImapStartTLSConnBroker) register(9500, AutoPop3StartTLSConnBroker) if socks is not None: register(1500, SocksConnBroker) register(3500, TorConnBroker) register(3500, TorRiskyBroker) register(3500, TorOnionBroker) # Note: At this point we have already imported security, which # also monkey-patches these same functions. This is a good # thing and is deliberate. :-) ssl.wrap_socket = monkey_patch(ssl.wrap_socket, SslWrapOnlyOnce) if hasattr(ssl, 'SSLContext'): ssl.SSLContext.wrap_socket = monkey_patch( ssl.SSLContext.wrap_socket, SslContextWrapOnlyOnce) from mailpile.plugins import PluginManager _plugins = PluginManager(builtin=__file__) _plugins.register_commands(NetworkHistory, GetTlsCertificate) else: import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/crypto/__init__.py ================================================ ================================================ FILE: mailpile/crypto/aes_utils.py ================================================ from __future__ import print_function # This is a compatibility wrapper for using whatever AES library is handy. # By default we support Cryptography and pyCrypto, with a preference for # Cryptography. # IMPORTANT: # # We currently only implement AES CTR mode, since this code is primarily # being used to write data to disk for long-term storage; the malleability # of CTR is considered a feature; if a bit gets flipped that doesn't destroy # all of the following blocks. # # This does mean we need to take special care with our IVs/nonces! # import os import struct from hashlib import md5 def make_cryptography_utils(): import os import cryptography.hazmat.backends from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes def _aes_ctr(key, nonce): # Notes: # # The funky business with the prefixed nonce is because the first # iteration of this code used pycrypto's Crypto.Util.Counter with # the prefix argument. Cryptography doesn't have such a counter API, # but if we carefully set the nonce we can achieve compatibility. # # The MD5 digests save the caller from having to know our internal # size requirements; AES wants 128, we just mix all the bits we're # given. We expect the input to already be strongly random (so MD5's # weaknesses shouldn't matter), but it may of the wrong size. # hashed_key = md5(key).digest() prefixed_nonce = md5(nonce).digest()[:8] + '\0\0\0\0\0\0\0\1' return Cipher( algorithms.AES(hashed_key), modes.CTR(prefixed_nonce), backend=cryptography.hazmat.backends.default_backend()) def aes_ctr_encryptor(key, nonce): return _aes_ctr(key, nonce).encryptor().update def aes_ctr_decryptor(key, nonce): return _aes_ctr(key, nonce).decryptor().update return aes_ctr_encryptor, aes_ctr_decryptor def make_pycrypto_utils(): from Crypto.Cipher import AES from Crypto.Util import Counter def _nonce_as_int(nonce): i1, i2, i3, i4 = struct.unpack(">IIII", nonce) return (i1 << 96 | i2 << 64 | i3 << 32 | i4) def _aes_ctr(key, nonce): # Notes: # # A previous iteration of this code used the Counter with a prefix, # which limited us to 2**64 iterations. This has been change to just # set an initial value and allow wraparound. # # The MD5 digests save the caller from having to know our internal # size requirements; AES wants 128, we just mix all the bits we're # given. We expect the input to already be strongly random (so MD5's # weaknesses shouldn't matter), but it may of the wrong size. # hashed_key = md5(key).digest() prefixed_nonce = md5(nonce).digest()[:8] + '\0\0\0\0\0\0\0\1' counter = Counter.new(128, initial_value=_nonce_as_int(prefixed_nonce)) return AES.new(hashed_key, mode=AES.MODE_CTR, counter=counter) def aes_ctr_encryptor(key, nonce): return _aes_ctr(key, nonce).encrypt def aes_ctr_decryptor(key, nonce): return _aes_ctr(key, nonce).decrypt return aes_ctr_encryptor, aes_ctr_decryptor def make_dummy_utils(): def aes_ctr_encryptor(key): return lambda d: d def aes_ctr_decryptor(key): return lambda d: d return aes_ctr_encryptor, aes_ctr_decryptor ############################################################################## try: aes_ctr_encryptor, aes_ctr_decryptor = make_cryptography_utils() except ImportError: try: aes_ctr_encryptor, aes_ctr_decryptor = make_pycrypto_utils() except ImportError: raise ImportError("Please pip install cryptography (or pycrypto)") def getrandbits(count): bits = os.urandom(count // 8) rint = 0 while bits: rint = (rint << 8) | struct.unpack("B", bits[0])[0] bits = bits[1:] return rint def aes_ctr_encrypt(key, iv, data): return aes_ctr_encryptor(key, iv)(data) def aes_ctr_decrypt(key, iv, data): return aes_ctr_decryptor(key, iv)(data) if __name__ == "__main__": import base64 bogus_key = "01234567890abcdef" bogus_nonce = "this is a bogus nonce that is bogus" hello = "hello world" results = [] for name, backend in (('Cryptography', make_cryptography_utils), ('pyCrypto', make_pycrypto_utils)): aes_ctr_encryptor, aes_ctr_decryptor = backend() ct1 = aes_ctr_encryptor(bogus_key, bogus_nonce)(hello) results.append((name, base64.b64encode(ct1))) ct2 = aes_ctr_encrypt(bogus_key, bogus_nonce, hello) results.append((name, base64.b64encode(ct2))) assert(aes_ctr_decrypt(bogus_key, bogus_nonce, ct1) == aes_ctr_decryptor(bogus_key, bogus_nonce)(ct1) == hello) # Make sure all the results are the same okay = True r1 = results[0] for result in results[1:]: if r1[1] != result[1]: print('%s != %s' % (r1, result)) okay = False assert(okay) # This verifies we can decrypt some snippets of data that were # generated with a previous iteration of mailpile.crypto.streamer from mailpile.util import sha512b64 as genkey legacy_data = "part two, yeaaaah\n" legacy_nonce = "2c1c43936034cae20eef86d961cb6570" legacy_key = genkey("test key", legacy_nonce)[:32].strip() legacy_ct = base64.b64decode("D+lBOPrtV+amUCAtoFPCzxsZ") decrypted = aes_ctr_decrypt(legacy_key, legacy_nonce, legacy_ct) assert(legacy_data == decrypted) print("ok") ================================================ FILE: mailpile/crypto/autocrypt.py ================================================ from __future__ import print_function # Copyright (C) 2018 Jack Dodds & Mailpile ehf. # This code is part of Mailpile and is hereby released under the # Gnu Affero Public Licence v.3 - see ../../COPYING and ../../AGPLv3.txt. # """ This file contains low-level Autocrypt code: constants, parsing, etc. The higher level application logic (state database, persistance etc.) is mostly in mailpile.plugins.crypto_autocrypt, but inevitably some has leaked into mailpile.crypto.gpgi and mailpile.crypto.mime. Examples (and doctests): # Canonicalize e-mail addresses according to Autocrypt conventions >>> canonicalize_email('BrE@maILPile.is') 'bre@mailpile.is' # Parse the Autocrypt header >>> ach = 'addr=bre@mailpile.is; _a=b; _c=d; keydata=aGVsbG8=' >>> hv = parse_autocrypt_headervalue(ach, optional_attrs=['_a']) >>> hv['addr'] 'bre@mailpile.is' >>> hv['keydata'] 'hello' >>> hv['_a'] 'b' >>> hv.get('_c') is None True >>> hv.get('prefer-encrypt') is None True # Invalid autocrypt headers return {} >>> parse_autocrypt_headervalue('addr=bre@mailpile.is') {} >>> parse_autocrypt_headervalue('keydata=aGVsbG8=') {} >>> parse_autocrypt_headervalue('unknown=attribute; ' + ach) {} # Invalid prefer-encrypt values just get ignored >>> hv = parse_autocrypt_headervalue('prefer-encrypt=bogus; ' + ach) >>> hv.get('prefer-encrypt') is None True >>> hv = parse_autocrypt_headervalue('prefer-encrypt=mutual; ' + ach) >>> hv.get('prefer-encrypt') == 'mutual' True # Generate a valid Autocrypt header >>> make_autocrypt_header('bre@mailpile.is', 'hello', prefer_encrypt_mutual=True) 'addr=bre@mailpile.is; prefer-encrypt=mutual; keydata=aGVsbG8=' # Autocrypt setup-codes are used to secure our PGP keys >>> generate_autocrypt_setup_code(random_data='fake random garbage data') '1189-1868-6510-5211-5608-1629-1262-5635-4164' >>> len(generate_autocrypt_setup_code()) 44 >>> generate_autocrypt_setup_code() != generate_autocrypt_setup_code() True # AutocryptRecommendations combine a key and a policy of what to do >>> ar = AutocryptRecommendation('disable') >>> ar.policy 'disable' >>> ar.key_sig is None True # Combining recommendations for multiple parties has specific rules >>> ar2 = AutocryptRecommendation('encrypt', key_sig='12345') >>> AutocryptRecommendation.Synchronize(ar, ar2) 'disable' >>> str(ar2) 'disable' # Not just anything is a valid recommendation >>> ar2.policy = 'bogus' Traceback (most recent call last): ... ValueError: Invalid Autocrypt policy: bogus """ import base64 import datetime import os import pgpdump import struct import time AUTOCRYPT_IGNORE_MIMETYPES = ('multipart/report', ) def canonicalize_email(address): try: localpart, domain = address.split('@') except (ValueError, AttributeError): # Just return invalid e-mails unchanged, there is no sensible way # to canonicalize such a thing. return address # FIXME: Ensure domain is ASCII, if not, punycode it domain = domain.lower() # FIXME: Ensure we're using the "empty locale" localpart = localpart.lower() # NOTE: We deliberately do not strip plussed parts or perform any other # normalization of the localpart beyond lowercasing. This is both # to comply with the Autocrypt Level 1 spec, but also because being able # to use plussed parts to allow differing cryptographic identities to # share the same e-mail account is something power users like to do. return '%s@%s' % (localpart, domain) def parse_autocrypt_headervalue(value, optional_attrs=None): # Based on: # # https://github.com/mailencrypt/inbome/blob/master/src/inbome/parse.py """ Parse an AutoCrypt header. Will return an empty dict if parsing fails. Optional attributes may be added to the result dictionary, but only the ones listed in optional_attrs (a list or dict); others are ignored. """ result_dict = {} try: for x in value.split(";"): kv = x.split("=", 1) name = kv[0].strip() value = kv[1].strip() if name in ("addr", "prefer-encrypt"): result_dict[name] = value elif name == "keydata": keydata_base64 = "".join(value.split()) keydata = base64.b64decode(keydata_base64) result_dict[name] = keydata elif name[:1] == '_': if optional_attrs and name in optional_attrs: result_dict[name] = value else: # Unknown value detected, refuse to parse any further return {} except (ValueError, TypeError, IndexError): return {} if "keydata" not in result_dict: # found no keydata, ignoring header return {} if "addr" not in result_dict: # found no e-mail address, ignoring header return {} else: result_dict["addr"] = canonicalize_email(result_dict["addr"]) if result_dict.get("prefer-encrypt") not in ("mutual", None): # Invalid prefer-encrypt value; treat as nopreference del result_dict['prefer-encrypt'] return result_dict def extract_autocrypt_header(msg, to=None, optional_attrs=None): # Autocrypt requires there only be one From header froms = msg.get_all("From") or [] if len(froms) != 1: return {} # Extract the from address for comparisons below. We compare the # canonicalized versions, which is not the strictest interpretation # of the spec, but feels like a reasonable balance here. from mailpile.mailutils.addresses import AddressHeaderParser from_addrs = AddressHeaderParser(froms[0]) if len(from_addrs) != 1: return {} from_addr = canonicalize_email(from_addrs[0].address) to = canonicalize_email(to) if to else None all_results = [] for inb in (msg.get_all("Autocrypt") or []): res = parse_autocrypt_headervalue(inb, optional_attrs=optional_attrs) if res: if ((not to or canonicalize_email(res['addr']) == to) and (canonicalize_email(res['addr']) == from_addr)): all_results.append(res) # Return parsed header iff we found exactly one. if len(all_results) == 1: return all_results[0] else: return {} def extract_autocrypt_gossip_headers(msg, to=None, optional_attrs=None): to = canonicalize_email(to) if to else None all_results = [] for inb in (msg.get_all("Autocrypt-Gossip") or []): res = parse_autocrypt_headervalue(inb, optional_attrs=optional_attrs) if res and (not to or res['addr'] == to): all_results.append(res) return all_results def make_autocrypt_header(addr, binary_key, prefer_encrypt_mutual=False, prefix='Autocrypt'): prefix = '%s: ' % prefix pem = ' prefer-encrypt=mutual;' if prefer_encrypt_mutual else '' hdr = '%saddr=%s;%s keydata=' % (prefix, addr, pem) for c in base64.b64encode(binary_key).strip(): if (len(hdr) % 78) == 0: hdr += ' ' hdr += c return hdr[len(prefix):] def generate_autocrypt_setup_code(random_data=None): """ Generate a passphrase/setup-code compliant with Autocrypt Level 1. From the spec: An Autocrypt Level 1 MUA MUST generate a Setup Code as UTF-8 string of 36 numeric characters, divided into nine blocks of four, separated by dashes. The dashes are part of the secret code and there are no spaces. This format holds about 119 bits of entropy. It is designed to be unambiguous, pronounceable, script-independent (Chinese, Cyrillic etc.), easily input on a mobile device and split into blocks that are easily kept in short term memory. """ random_data = random_data or os.urandom(16) # 16 bytes = 128 bits entropy ints = struct.unpack('>4I', random_data[:16]) ival = ints[0] + (ints[1] << 32) + (ints[2] << 64) + (ints[3] << 96) blocks = [] while len(blocks) < 9: blocks.append('%4.4d' % (ival % 10000)) ival //= 10000 return '-'.join(blocks) # FIXME: Add a with_signing_subkeys=True, implement. This deviates # from the Autocrypt spec, because Autocrypt says nothing about # signatures. But we're almost always signing our mail, and w/o # the subkeys the signatures cannot be checked. def UNUSED_get_minimal_PGP_key(keydata, user_id=None, subkey_id=None, binary_out=False): """ Accepts a PGP key (armored or binary) and returns a minimal PGP key containing exactly five packets (base64 or binary) defining a primary key, a single user id with one self-signature, and a single encryption subkey with one self-signature. Such a five packet key MUST be used in Autocrypt headers (Level 1 Spec section 2.1.1). The unrevoked user id with newest unexpired self-signature and the unrevoked encryption-capable subkey with newest unexpired self-signature are selected from the input key. If user_id is provided, a user id containing that string will be selected if there is one, otherwise any user id will be accepted. If subkey_id is specified, only a subkey with that id will be selected. Along with the new key, the selected user id and subkey id are returned. Returns None if there is a failure. """ def _get_int4(data, offset): '''Pull four bytes from data at offset and return as an integer.''' return ((data[offset] << 24) + (data[offset + 1] << 16) + (data[offset + 2] << 8) + data[offset + 3]) def _exp_time(creation_time, exp_time_subpacket_data): life_s = _get_int4(exp_time_subpacket_data, 0) if not life_s: return 0 return packet.creation_time + datetime.timedelta( seconds = life_s) def _pgp_header(type, body_length): if body_length < 192: return bytearray([type+0xC0, body_length]) elif body_length < 8384: return bytearray([type+0xC0, (body_length-192)//256+192, (body_length-192)%256]) else: return bytearray([type+0xC0, 255, body_length//(1<<24), body_length//(1<<16) % 256, body_length//1<<8 % 256, body_length % 256]) pri_key = None u_id = None u_id_sig = None u_id_match = False s_key = None s_key_sig = None user_id = canonicalize_email(user_id) if user_id else None now = datetime.datetime.utcfromtimestamp(time.time()) if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in keydata: packet_iter = pgpdump.AsciiData(keydata).packets() else: packet_iter = pgpdump.BinaryData(keydata).packets() try: packet = next(packet_iter) except: packet = None while packet: if packet.raw == 6 and pri_key: # Primary key must be the first break # and only the first packet. elif packet.raw != 6 and not pri_key: break elif packet.raw == 6: # Primary Public-Key Packet pri_key = packet elif packet.raw == 13: # User ID Packet u_id_try = packet u_id_sig_try = None u_id_try_match = ( not user_id or (user_id == canonicalize_email(u_id_try.user))) # Accept a nonmatching u_id IFF no other u_id matches. if u_id_match and not u_id_try_match: u_id_try = None for packet in packet_iter: if packet.raw != 2: # Signature Packet break elif not u_id_try: continue # User ID certification elif packet.raw_sig_type in (0x10, 0x11, 0x12, 0x13, 0x1F): if (pri_key.fingerprint.endswith(packet.key_id) and (not packet.expiration_time or packet.expiration_time > now) and (not u_id_sig_try or u_id_sig_try.creation_time < packet.creation_time)): u_id_sig_try = packet # Certification revocation elif packet.raw_sig_type == 0x30: if pri_key.fingerprint.endswith(packet.key_id): u_id_try = None u_id_sig_try = None # Select unrevoked user id with newest unexpired self-signature if u_id_try and u_id_sig_try and ( not u_id or not u_id_sig or u_id_try_match and not u_id_match or u_id_sig_try.creation_time >= u_id_sig.creation_time): u_id = u_id_try u_id_sig = u_id_sig_try u_id_match = u_id_try_match continue # Skip next(packet_iter) - for has done it. elif packet.raw == 14: # Public-Subkey Packet s_key_try = packet s_key_sig_try = None # Honour a request for specific subkey and check for expiry. if ((subkey_id and not s_key_try.fingerprint.endswith(subkey_id)) or (s_key_try.expiration_time and s_key_try.expiration_time < now)): s_key_try = None for packet in packet_iter: if packet.raw != 2: # Signature Packet break elif not s_key_try: continue # Subkey Binding Signature elif packet.raw_sig_type == 0x18: packet.key_expire_time = None if (pri_key.fingerprint.endswith(packet.key_id) and not packet.expiration_time or packet.expiration_time >= now): can_encrypt = True # Assume encrypt -- FIXME for subpacket in packet.subpackets: if subpacket.subtype == 9: # Key expiration packet.key_expire_time = _exp_time( packet.creation_time, subpacket.data) elif subpacket.subtype == 27: # Key flags can_encrypt |= subpacket.data[0] & 0x0C if can_encrypt and (not packet.key_expire_time or packet.key_expire_time >= now): s_key_sig_try = packet # Subkey revocation signature elif packet.raw_sig_type == 0x28: if pri_key.fingerprint.endswith(packet.key_id): s_key_try = None s_key_sig_try = None # Select unrevoked encryption-capable subkey with newest # unexpired self-signature (ignores newness of key itself). if s_key_try and s_key_sig_try and (not s_key_sig or s_key_sig_try.creation_time >= s_key_sig.creation_time): s_key = s_key_try s_key_sig = s_key_sig_try continue # Skip next(packet_iter) - for has done it. try: packet = next(packet_iter) except: packet = None if not(pri_key and u_id and u_id_sig and s_key and s_key_sig): return '', None, None newkey = ( _pgp_header(pri_key.raw, len(pri_key.data)) + pri_key.data + _pgp_header(u_id.raw, len(u_id.data)) + u_id.data + _pgp_header(u_id_sig.raw, len(u_id_sig.data)) + u_id_sig.data + _pgp_header(s_key.raw, len(s_key.data)) + s_key.data + _pgp_header(s_key_sig.raw, len(s_key_sig.data)) + s_key_sig.data ) if not binary_out: newkey = base64.b64encode(newkey) return newkey, u_id.user, s_key.key_id class AutocryptRecommendation(object): DISABLE = "disable" DISCOURAGE = "discourage" ENABLE = "enable" ENCRYPT = "encrypt" ORDERED_POLICIES = (DISABLE, DISCOURAGE, ENABLE, ENCRYPT) def __init__(self, policy, key_sig=None): self.key_sig = self._policy = None self.set_recommendation(policy, key_sig) def __str__(self): if self.policy in (self.DISABLE,): return self.policy return "%s (key=%s)" % (self.policy, self.key_sig) @classmethod def Synchronize(cls, *recommendations): """ This will synchronize a set of Autocrypt recommendations to whatever the lowest common denomitor is, and then return that policy. """ if not recommendations: return cls.DISABLE lowest_common_policy = cls.ORDERED_POLICIES[min( cls.ORDERED_POLICIES.index(r.policy) for r in recommendations)] for r in recommendations: r.policy = lowest_common_policy return lowest_common_policy def set_recommendation(self, policy, key_sig=None): if policy not in self.ORDERED_POLICIES: raise ValueError('Invalid Autocrypt policy: %s' % policy) if policy != self.DISABLE and key_sig is None and self.key_sig is None: raise ValueError('Policy %s requires a key' % policy) self._policy = policy if key_sig is not None: self.key_sig = key_sig policy = property(lambda self: self._policy, set_recommendation) if __name__ == "__main__": import sys import doctest results = doctest.testmod(optionflags=doctest.ELLIPSIS) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/crypto/gpgi.py ================================================ #coding:utf-8 from __future__ import print_function import os import string import sys import time import re import StringIO import tempfile import threading import traceback import urllib import select import pgpdump import pgpdump.utils import base64 import quopri from datetime import datetime from email.parser import Parser from email.message import Message from threading import Thread import mailpile.platforms from mailpile.i18n import gettext from mailpile.i18n import ngettext as _n from mailpile.crypto.state import * from mailpile.crypto.mime import MimeSigningWrapper, MimeEncryptingWrapper from mailpile.safe_popen import Popen, PIPE, Safe_Pipe _ = lambda s: s DEFAULT_KEYSERVERS = ["hkps://hkps.pool.sks-keyservers.net", "hkp://subset.pool.sks-keyservers.net"] DEFAULT_KEYSERVER_OPTIONS = [ 'ca-cert-file=%s' % __file__.replace('.pyc', '.py')] DEFAULT_IMPORT_OPTIONS = [ 'import-minimal'] GPG_KEYID_LENGTH = 8 GNUPG_HOMEDIR = None # None=use what gpg uses GPG_BINARY = mailpile.platforms.GetDefaultGnuPGCommand GPG_VERSIONS = {} BLOCKSIZE = 65536 openpgp_algorithms = {1: _("RSA"), 2: _("RSA (encrypt only)"), 3: _("RSA (sign only)"), 16: _("ElGamal (encrypt only)"), 17: _("DSA"), 20: _("ElGamal (encrypt/sign) [COMPROMISED]"), 22: _("EdDSA"), 999: _("Unknown")} # For details on type 20 compromisation, see # http://lists.gnupg.org/pipermail/gnupg-announce/2003q4/000160.html ENTROPY_LOCK = threading.Lock() class GnuPGEventUpdater: """ Parse the GPG response into something useful for the Event Log. """ def __init__(self, event): from mailpile.eventlog import Event self.event = event or Event() def _log(self, section, message): data = section.get('gnupg', []) if data: data[-1].append(message) def _log_private(self, message): self._log(self.event.private_data, message) def _log_public(self, message): self._log(self.event.private_data, message) self._log(self.event.data, message) def running_gpg(self, why): for section in (self.event.data, self.event.private_data): data = section.get('gnupg', []) data.append([why, int(time.time())]) section['gnupg'] = data def update_args(self, args): self._log_public(' '.join(args)) def update_sent_passphrase(self): self._log_public('Sent passphrase') def _parse_gpg_line(self, line): if line.startswith('[GNUPG:] '): pass # FIXME: Parse for machine-readable data elif line.startswith('gpg: '): self._log_private(line[5:].strip()) def update_stdout(self, line): self._parse_gpg_line(line) def update_stderr(self, line): self._parse_gpg_line(line) def update_return_code(self, code): self._log_public('GnuPG returned %s' % code) class GnuPGResultParser: """ Parse the GPG response into EncryptionInfo and SignatureInfo. """ def __init__(rp, decrypt_requires_MDC=True, debug=None): rp.decrypt_requires_MDC = decrypt_requires_MDC rp.debug = debug or (lambda t: True) rp.signature_info = SignatureInfo() rp.signature_info["protocol"] = "openpgp" rp.encryption_info = EncryptionInfo() rp.encryption_info["protocol"] = "openpgp" rp.plaintext = "" def parse(rp, retvals): signature_info = rp.signature_info encryption_info = rp.encryption_info from mailpile.mailutils.emails import ExtractEmailAndName # Belt & suspenders: work around some buggy GnuPG status codes gpg_stderr = ''.join(retvals[1]["stderr"]) # First pass, set some initial state. locked, missing = [], [] for data in retvals[1]["status"]: keyword = data[0].strip() # The last keyword often ends in \n if keyword == 'NEED_PASSPHRASE': locked += [data[2]] encryption_info.part_status = "lockedkey" encryption_info["locked_keys"] = list(set(locked)) elif keyword == 'GOOD_PASSPHRASE': encryption_info["locked_keys"] = [] elif keyword == "DECRYPTION_FAILED": missing += [x[1].strip() for x in retvals[1]["status"] if x[0] == "NO_SECKEY"] if missing: encryption_info["missing_keys"] = list(set(missing)) if encryption_info.part_status != "lockedkey": if missing: encryption_info.part_status = "missingkey" else: encryption_info.part_status = "error" elif keyword == "DECRYPTION_OKAY": if (rp.decrypt_requires_MDC and 'message was not integrity protected' in gpg_stderr): rp.debug('Message not integrity protected, failing.') encryption_info.part_status = "error" else: encryption_info.part_status = "decrypted" rp.plaintext = "".join(retvals[1]["stdout"]) elif keyword == "ENC_TO": keylist = encryption_info.get("have_keys", []) if data[1] not in keylist: keylist.append(data[1].strip()) encryption_info["have_keys"] = list(set(keylist)) elif keyword == "PLAINTEXT": encryption_info.filename = data[3].strip() elif signature_info.part_status == "none": # Only one of these will ever be emitted per key, use # this to set initial state. We may end up revising # the status depending on more info later. if keyword in ("GOODSIG", "BADSIG"): email, fn = ExtractEmailAndName( " ".join(data[2:]).decode('utf-8')) signature_info["name"] = fn signature_info["email"] = email signature_info.part_status = ((keyword == "GOODSIG") and "unverified" or "invalid") rp.plaintext = "".join(retvals[1]["stdout"]) elif keyword == "ERRSIG": signature_info.part_status = "error" signature_info["keyinfo"] = data[1] signature_info["timestamp"] = int(data[5]) # Second pass, this may update/mutate the state set above for data in retvals[1]["status"]: keyword = data[0].strip() # The last keyword often ends in \n if keyword == "NO_SECKEY": keyid = data[1].strip() if "missing_keys" not in encryption_info: encryption_info["missing_keys"] = [keyid] elif keyid not in encryption_info["missing_keys"]: encryption_info["missing_keys"].append(keyid) while keyid in encryption_info["have_keys"]: encryption_info["have_keys"].remove(keyid) elif keyword == "VALIDSIG": # FIXME: Determine trust level, between new, unverified, # verified, untrusted. signature_info["keyinfo"] = data[1] signature_info["timestamp"] = int(data[3]) elif keyword in ("EXPKEYSIG", "REVKEYSIG"): email, fn = ExtractEmailAndName( " ".join(data[2:]).decode('utf-8')) signature_info["name"] = fn signature_info["email"] = email signature_info.part_status = ((keyword == "EXPKEYSIG") and "expired" or "revoked") # FIXME: This appears to be spammy. Is my key borked, or # is GnuPG being stupid? # # elif keyword == "KEYEXPIRED": # Ignoring: SIGEXPIRED # signature_info.part_status = "expired" elif keyword == "KEYREVOKED": signature_info.part_status = "revoked" elif keyword == "NO_PUBKEY": signature_info.part_status = "unknown" elif keyword in ("TRUST_ULTIMATE", "TRUST_FULLY"): if signature_info.part_status == "unverified": signature_info.part_status = "verified" elif (keyword == "DECRYPTION_INFO" and encryption_info.part_status == "decrypted" and rp.decrypt_requires_MDC): mdc_method = data[1].strip() aead_algo = data[3].strip() if len(data) > 3 else 0 if not mdc_method and not aeadalgo: encryption_info.part_status = "error" if encryption_info.part_status == "error": rp.plaintext = "" return rp class GnuPGRecordParser: def __init__(self): self.keys = {} self.curkeyid = None self.curdata = None self.record_fields = ["record", "validity", "keysize", "keytype", "keyid", "creation_date", "expiration_date", "uidhash", "ownertrust", "uid", "sigclass", "capabilities", "flag", "sn", "hashtype", "curve"] self.record_types = ["pub", "sub", "ssb", "fpr", "uat", "sec", "tru", "sig", "rev", "uid", "gpg", "rvk", "grp"] self.record_parsers = [self.parse_pubkey, self.parse_subkey, self.parse_subkey, self.parse_fingerprint, self.parse_userattribute, self.parse_privkey, self.parse_trust, self.parse_signature, self.parse_revoke, self.parse_uidline, self.parse_none, self.parse_revocation_key, self.parse_keygrip] self.dispatch = dict(zip(self.record_types, self.record_parsers)) def parse(self, lines): for line in lines: self.parse_line(line) return self.keys def parse_line(self, line): line = dict(zip(self.record_fields, map(lambda s: s.replace("\\x3a", ":"), stubborn_decode(line).strip().split(":")))) r = self.dispatch.get(line["record"], self.parse_unknown) r(line) def _parse_dates(self, line): for ts in ('expiration_date', 'creation_date'): if line.get(ts) and '-' not in line[ts]: line[ts+'_ts'] = line[ts] try: unixtime = int(line[ts]) if unixtime > 946684800: # 2000-01-01 dt = datetime.fromtimestamp(unixtime) line[ts] = dt.strftime('%Y-%m-%d') except ValueError: line[ts] = '1970-01-01' def _parse_keydata(self, line): line["keytype_name"] = _(openpgp_algorithms.get(int(line["keytype"]), 'Unknown')) line["capabilities_map"] = { "encrypt": "E" in line["capabilities"], "sign": "S" in line["capabilities"], "certify": "C" in line["capabilities"], "authenticate": "A" in line["capabilities"], } line["disabled"] = "D" in line["capabilities"] line["revoked"] = "r" in line["validity"] line["expired"] = "e" in line["validity"] self._parse_dates(line) return line def _clean_curdata(self): for v in self.curdata.keys(): if self.curdata[v] == "": del self.curdata[v] del self.curdata["record"] def parse_pubkey(self, line): self.curkeyid = line["keyid"] self.curdata = self.keys[self.curkeyid] = self._parse_keydata(line) self.curdata["subkeys"] = [] self.curdata["uids"] = [] self.curdata["secret"] = (self.curdata["record"] == "sec") self.parse_uidline(self.curdata) self._clean_curdata() def parse_subkey(self, line): self.curdata = self._parse_keydata(line) self.keys[self.curkeyid]["subkeys"].append(self.curdata) self._clean_curdata() def parse_fingerprint(self, line): fpr = line["uid"] self.curdata["fingerprint"] = fpr if len(self.curkeyid) < len(fpr): self.keys[fpr] = self.keys[self.curkeyid] del(self.keys[self.curkeyid]) self.curkeyid = fpr def parse_userattribute(self, line): # TODO: We are currently ignoring user attributes as not useful. # We may at some point want to use --attribute-fd and read # in user photos and such? pass def parse_privkey(self, line): self.parse_pubkey(line) def parse_uidline(self, line): email, name, comment = parse_uid(line["uid"]) self._parse_dates(line) if email or name or comment: self.keys[self.curkeyid]["uids"].append({ "email": email, "name": name, "comment": comment, "creation_date": line["creation_date"] }) else: pass # This is the case where a uid or sec line have no # information aside from the creation date, which we # parse elsewhere. As these lines are effectively blank, # we omit them to simplify presentation to the user. def parse_trust(self, line): # FIXME: We are currently ignoring commentary from the Trust DB. pass def parse_signature(self, line): # FIXME: This is probably wrong; signatures are on UIDs and not # the key itself. No? Yes? Figure this out. if "signatures" not in self.keys[self.curkeyid]: self.keys[self.curkeyid]["signatures"] = [] sig = { "signer": line[9], "signature_date": line[5], "keyid": line[4], "trust": line[10], "keytype": line[4] } self.keys[self.curkeyid]["signatures"].append(sig) def parse_keygrip(self, line): self.curdata["keygrip"] = line["uid"] def parse_revoke(self, line): pass # FIXME def parse_revocation_key(self, line): pass # FIXME def parse_unknown(self, line): print("Unknown line with code '%s'" % (line,)) def parse_none(line): pass UID_PARSE_RE = "^([^\(\<]+?){0,1}( \((.+?)\)){0,1}( \<(.+?)\>){0,1}\s*$" def stubborn_decode(text): if isinstance(text, unicode): return text try: return text.decode("utf-8") except UnicodeDecodeError: try: return text.decode("iso-8859-1") except UnicodeDecodeError: return uidstr.decode("utf-8", "replace") def parse_uid(uidstr): matches = re.match(UID_PARSE_RE, uidstr) if matches: email = matches.groups(0)[4] or "" comment = matches.groups(0)[2] or "" name = matches.groups(0)[0] or "" else: if '@' in uidstr and ' ' not in uidstr: email, name = uidstr, "" else: email, name = "", uidstr comment = "" return email, name, comment class StreamReader(Thread): def __init__(self, name, fd, callback, lines=True): Thread.__init__(self, target=self.readin, args=(fd, callback)) self.name = name self.state = 'startup' self.lines = lines self.start() def __str__(self): return '%s(%s/%s, lines=%s)' % (Thread.__str__(self), self.name, self.state, self.lines) def readin(self, fd, callback): try: if self.lines: self.state = 'read' for line in iter(fd.readline, b''): self.state = 'callback' callback(line) self.state = 'read' else: while True: self.state = 'read' buf = fd.read(BLOCKSIZE) self.state = 'callback' callback(buf) if buf == "": break except: traceback.print_exc() finally: self.state = 'done' fd.close() class StreamWriter(Thread): def __init__(self, name, fd, output, partial_write_ok=False): Thread.__init__(self, target=self.writeout, args=(fd, output)) self.name = name self.state = 'startup' self.partial_write_ok = partial_write_ok self.start() def __str__(self): return '%s(%s/%s)' % (Thread.__str__(self), self.name, self.state) def writeout(self, fd, output): if isinstance(output, (str, unicode, bytearray)): total = len(output) output = StringIO.StringIO(output) else: total = 0 try: while True: self.state = 'read' line = output.read(BLOCKSIZE) if line == "": break self.state = 'write' fd.write(line) total -= len(line) output.close() except: if not self.partial_write_ok: print('%s: %s bytes left' % (self, total)) traceback.print_exc() finally: self.state = 'done' fd.close() DEBUG_GNUPG = False class GnuPG: """ Wrap GnuPG and make all functionality feel Pythonic. """ ARMOR_BEGIN_SIGNED = '-----BEGIN PGP SIGNED MESSAGE-----' ARMOR_BEGIN_SIGNATURE = '-----BEGIN PGP SIGNATURE-----' ARMOR_END_SIGNED = '-----END PGP SIGNATURE-----' ARMOR_END_SIGNATURE = '-----END PGP SIGNATURE-----' ARMOR_BEGIN_ENCRYPTED = '-----BEGIN PGP MESSAGE-----' ARMOR_END_ENCRYPTED = '-----END PGP MESSAGE-----' ARMOR_BEGIN_PUB_KEY = '-----BEGIN PGP PUBLIC KEY BLOCK-----' ARMOR_END_PUB_KEY = '-----END PGP PUBLIC KEY BLOCK-----' LAST_KEY_USED = 'DEFAULT' # This is a 1-value global cache def __init__(self, config, session=None, use_agent=None, debug=False, dry_run=False, event=None, passphrase=None): global DEBUG_GNUPG self.available = None self.outputfds = ["stdout", "stderr", "status"] self.errors = [] self.event = GnuPGEventUpdater(event) self.session = session self.config = config or (session and session.config) or None self.status_filenames = [] if self.config: DEBUG_GNUPG = ('gnupg' in self.config.sys.debug) self.homedir = self.config.sys.gpg_home or GNUPG_HOMEDIR self.gpgbinary = self.config.sys.gpg_binary or GPG_BINARY() self.passphrases = self.config.passphrases self.passphrase = (passphrase if (passphrase is not None) else self.passphrases['DEFAULT']).get_reader() self.use_agent = (use_agent if (use_agent is not None) else self.config.prefs.gpg_use_agent) else: self.homedir = GNUPG_HOMEDIR self.gpgbinary = GPG_BINARY() self.passphrases = None if passphrase: self.passphrase = passphrase.get_reader() else: self.passphrase = None self.use_agent = use_agent self.dry_run = dry_run self.debug = (self._debug_all if (debug or DEBUG_GNUPG) else self._debug_none) def prepare_passphrase(self, keyid, signing=False, decrypting=False): """Query the Mailpile secrets for a usable passphrase.""" def _use(kid, sps_reader): self.passphrase = sps_reader GnuPG.LAST_KEY_USED = kid return True if self.config: message = [] if decrypting: message.append(_("Your PGP key is needed for decrypting.")) if signing: message.append(_("Your PGP key is needed for signing.")) match, sps = self.config.get_passphrase(keyid, prompt=_('Unlock your encryption key'), description=' '.join(message)) if match: return _use(match, sps.get_reader()) self.passphrase = None # This *may* allow use of the GnuPG agent return False def _debug_all(self, msg): if self.session: self.session.debug(msg.rstrip()) else: print('%s' % str(msg).rstrip()) def _debug_none(self, msg): pass def set_home(self, path): self.homedir = path def version(self): """Returns a string representing the GnuPG version number.""" self.event.running_gpg(_('Checking GnuPG version')) retvals = self.run(["--version"], novercheck=True) return retvals[1]["stdout"][0].split('\n')[0] def version_tuple(self, update=False): """Returns a tuple representing the GnuPG version number.""" global GPG_VERSIONS if update or not GPG_VERSIONS.get(self.gpgbinary): match = re.search( "(\d+).(\d+).(\d+)", self.version() ) version = tuple(int(v) for v in match.groups()) GPG_VERSIONS[self.gpgbinary] = version return GPG_VERSIONS[self.gpgbinary] def gnupghome(self): """Returns the location of the GnuPG keyring""" self.event.running_gpg(_('Checking GnuPG home directory')) rv = self.run(["--version"], novercheck=True)[1]["stdout"][0] for l in rv.splitlines(): if l.startswith('Home: '): return os.path.expanduser(l[6:].strip()) return os.path.expanduser(os.getenv('GNUPGHOME', '~/.gnupg')) def is_available(self): try: self.event.running_gpg(_('Checking GnuPG availability')) self.version_tuple(update=True) self.available = True except OSError: self.available = False return self.available def common_args(self, args=None, version=None, will_send_passphrase=False, interactive=False): if args is None: args = [] if version is None: version = self.version_tuple() args.insert(0, self.gpgbinary) args.insert(1, "--utf8-strings") # Disable SHA1 and compression in all things GnuPG args[1:1] = ["--personal-digest-preferences=SHA512", "--personal-compress-preferences=Uncompressed", "--digest-algo=SHA512", "--cert-digest-algo=SHA512"] if self.homedir: args.insert(1, "--homedir=%s" % self.homedir) if version > (2, 1, 11): binaries = mailpile.platforms.DetectBinaries() for which, setting in (('GnuPG_dirmngr', 'dirmngr-program'), ('GnuPG_agent', 'agent-program')): if which in binaries: args.insert(1, "--%s=%s" % (setting, binaries[which])) else: print('wtf: %s not in %s' % (which, binaries)) if (not self.use_agent) or will_send_passphrase: if version < (1, 5): args.insert(1, "--no-use-agent") elif version > (2, 1, 11): args.insert(1, "--pinentry-mode=loopback") else: raise ImportError('Mailpile requires GnuPG 1.4.x or 2.1.12+ !') if not interactive: args.insert(1, "--with-colons") args.insert(1, "--verbose") args.insert(1, "--batch") args.insert(1, "--enable-progress-filter") if self.status_filenames: args.insert(1, "--status-file=%s" % self.status_filenames[-1]) if will_send_passphrase: args.insert(2, "--passphrase-fd=0") if self.dry_run: args.insert(1, "--dry-run") return args def run(self, *args, **kwargs): # This wrapper handles temporary status files. Since we may recursively # invoke ourselves, we keep a stack of tempfiles and push/pop from the # list. fd = tempfile.NamedTemporaryFile(delete=False) fd.close() # Avoid potential conflicts on Windows try: self.status_filenames.append(fd.name) return self.run_without_status(*args, **kwargs) finally: os.remove(self.status_filenames.pop(-1)) def run_without_status(self, args=None, gpg_input=None, outputfd=None, partial_read_ok=False, send_passphrase=False, _raise=None, novercheck=False): if novercheck: version = (1, 4) else: version = self.version_tuple() args = self.common_args( args=list(args if args else []), version=version, will_send_passphrase=(self.passphrase and send_passphrase)) self.outputbuffers = dict([(x, []) for x in self.outputfds]) self.threads = {} gpg_retcode = -1 proc = None try: if send_passphrase and (self.passphrase is None): self.debug('Running WITHOUT PASSPHRASE %s' % ' '.join(args)) self.debug(''.join(traceback.format_stack())) else: self.debug('Running %s' % ' '.join(args)) # Here we go! self.event.update_args(args) proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=0) # GnuPG is a bit crazy, and requires that the passphrase # be sent and the filehandle closed before anything else # interesting happens. if send_passphrase and self.passphrase is not None: self.passphrase.seek(0, 0) c = self.passphrase.read(BLOCKSIZE) while c != '': proc.stdin.write(c) c = self.passphrase.read(BLOCKSIZE) proc.stdin.write('\n') self.event.update_sent_passphrase() wtf = ' '.join(args) self.threads = { "stderr": StreamReader('gpgi-stderr(%s)' % wtf, proc.stderr, self.parse_stderr) } if outputfd: self.threads["stdout"] = StreamReader( 'gpgi-stdout-to-fd(%s)' % wtf, proc.stdout, outputfd.write, lines=False) else: self.threads["stdout"] = StreamReader( 'gpgi-stdout-parsed(%s)' % wtf, proc.stdout, self.parse_stdout) if gpg_input: # If we have output, we just stream it. Technically, this # doesn't really need to be a thread at the moment. self.debug('< 1: print('WARNING: Failed to reap thread %s' % thr) def parse_status(self, line, *args): self.debug('<>> g = GnuPG(None) >>> g.list_keys()[0] 0 """ list_keys = ["--fingerprint"] for sel in set(selectors or []): list_keys += ["--list-keys", sel] if not selectors: list_keys += ["--list-keys"] self.event.running_gpg(_('Fetching GnuPG public key list (selectors=%s)' ) % ', '.join(selectors or [])) retvals = self.run(list_keys) return self.parse_keylist(retvals[1]["stdout"]) def list_secret_keys(self, selectors=None): # # Note: The selectors that are passed by default work around a bug # in GnuPG < 2.1, where --list-secret-keys does not list # details about key capabilities or expiry for # --list-secret-keys unless a selector is provided. A dot # is reasonably likely to appear in all PGP keys, as it is # a common component of e-mail addresses (and @ does not # work as a selector for some reason...) # # The downside of this workaround is that keys with no e-mail # address or an address like alice@localhost won't be found. # So we disable this hack on GnuPG >= 2.1. # if not selectors and self.version_tuple() < (2, 1): selectors = [".", "a", "e", "i", "p", "t", "k"] list_keys = ["--fingerprint"] if selectors: for sel in selectors: # FIXME - In 2.1.18 and 1.4.21 only one --list-keys is needed. list_keys += ["--list-secret-keys", sel] else: list_keys += ["--list-secret-keys"] self.event.running_gpg(_('Fetching GnuPG secret key list (selectors=%s)' ) % ', '.join(selectors or ['None'])) retvals = self.run(list_keys) secret_keys = self.parse_keylist(retvals[1]["stdout"]) # Another unfortunate thing GnuPG < 2.1 does, is it hides the disabled # state when listing secret keys; it seems internally only the # public key is disabled. This makes it hard for us to reason about # which keys can actually be used, so we compensate... list_keys = ["--fingerprint"] for fprint in set(secret_keys): # FIXME - In both 2.1.18 and 1.4.21 only one --list-keys is needed. list_keys += ["--list-keys", fprint] retvals = self.run(list_keys) public_keys = self.parse_keylist(retvals[1]["stdout"]) for fprint, info in public_keys.iteritems(): if fprint in set(secret_keys): for k in ("disabled", "revoked", "expired"): secret_keys[fprint][k] = info[k] return secret_keys def import_keys(self, key_data=None, import_options=DEFAULT_IMPORT_OPTIONS, filter_uid_emails=None): """ Imports gpg keys from a file object or string. >>> key_data = open("testing/pub.key").read() >>> g = GnuPG(None) >>> g.import_keys(key_data) {'failed': [], 'updated': [{'details_text': 'unchanged', 'details': 0, 'fingerprint': '08A650B8E2CBC1B02297915DC65626EED13C70DA'}], 'imported': [], 'results': {'sec_dups': 0, 'unchanged': 1, 'num_uids': 0, 'skipped_new_keys': 0, 'no_userids': 0, 'num_signatures': 0, 'num_revoked': 0, 'sec_imported': 0, 'sec_read': 0, 'not_imported': 0, 'count': 1, 'imported_rsa': 0, 'imported': 0, 'num_subkeys': 0}} """ self.event.running_gpg(_('Importing key to GnuPG key chain')) cmd = ["--import"] if filter_uid_emails: expr = ' || '.join('uid =~ %s' % e for e in filter_uid_emails) cmd[1:1] = ['--import-filter', 'keep-uid=%s' % expr] for opt in import_options: cmd[1:1] = ['--import-options', opt] retvals = self.run(cmd, gpg_input=key_data) return self._parse_import(retvals[1]["status"]) def _parse_import(self, output): res = {"imported": [], "updated": [], "failed": []} for x in output: if x[0] == "IMPORTED": res["imported"].append({ "fingerprint": x[1], "username": x[2].rstrip() }) elif x[0] == "IMPORT_OK": reasons = { "0": "unchanged", "1": "new key", "2": "new user IDs", "4": "new signatures", "8": "new subkeys", "16": "contains private key", "17": "contains new private key", } res["updated"].append({ "details": int(x[1]), # FIXME: Reasons may be ORed! This does NOT handle that. "details_text": reasons.get(x[1], str(x[1])), "fingerprint": x[2].rstrip(), }) elif x[0] == "IMPORT_PROBLEM": reasons = { "0": "no reason given", "1": "invalid certificate", "2": "issuer certificate missing", "3": "certificate chain too long", "4": "error storing certificate", } res["failed"].append({ "details": int(x[1]), "details_text": reasons.get(x[1], str(x[1])), "fingerprint": x[2].rstrip() }) elif x[0] == "IMPORT_RES": res["results"] = { "count": int(x[1]), "no_userids": int(x[2]), "imported": int(x[3]), "imported_rsa": int(x[4]), "unchanged": int(x[5]), "num_uids": int(x[6]), "num_subkeys": int(x[7]), "num_signatures": int(x[8]), "num_revoked": int(x[9]), "sec_read": int(x[10]), "sec_imported": int(x[11]), "sec_dups": int(x[12]), "skipped_new_keys": int(x[13]), "not_imported": int(x[14].rstrip()), } return res def decrypt(self, data, outputfd=None, passphrase=None, as_lines=False, require_MDC=True): """ Note that this test will fail if you don't replace the recipient with one whose key you control. >>> g = GnuPG(None) >>> ct = g.encrypt("Hello, World", to=["smari@mailpile.is"])[1] >>> g.decrypt(ct)["text"] 'Hello, World' """ if passphrase is not None: self.passphrase = passphrase.get_reader() elif GnuPG.LAST_KEY_USED: # This is an opportunistic approach to passphrase usage... we # just hope the passphrase we used last time will work again. # If we are right, we are done. If we are wrong, the output # will tell us which key IDs to look for in our secret stash. self.prepare_passphrase(GnuPG.LAST_KEY_USED, decrypting=True) self.event.running_gpg(_('Decrypting %d bytes of data') % len(data)) for tries in (1, 2): retvals = self.run(["--decrypt"], gpg_input=data, outputfd=outputfd, send_passphrase=True) if tries == 1: keyid = None for msg in reversed(retvals[1]['status']): # Reverse order so DECRYPTION_OKAY overrides KEY_CONSIDERED. # If decryption is not ok, look for good passphrase, retry. if msg[0] == 'DECRYPTION_OKAY': break elif (msg[0] == 'NEED_PASSPHRASE') and (passphrase is None): # This message is output by gpg 1.4 but not 2.1. if self.prepare_passphrase(msg[2], decrypting=True): keyid = msg[2] break elif (msg[0] == 'KEY_CONSIDERED') and (passphrase is None): # This message is output by gpg 2.1 but not 1.4. if self.prepare_passphrase(msg[1], decrypting=True): keyid = msg[1] break if not keyid: break if as_lines: as_lines = retvals[1]["stdout"] retvals[1]["stdout"] = [] rp = GnuPGResultParser(decrypt_requires_MDC=require_MDC, debug=self.debug).parse(retvals) return (rp.signature_info, rp.encryption_info, as_lines or rp.plaintext) def base64_segment(self, dec_start, dec_end, skip, line_len, line_end = 2): """ Given the start and end index of a desired segment of decoded data, this function finds smallest segment of an encoded base64 array that when decoded will include the desired decoded segment. It's assumed that the base64 data has a uniform line structure of line_len encoded characters including line_end eol characters, and that there are skip header characters preceding the base64 data. """ enc_start = 4*(dec_start/3) dec_skip = dec_start - 3*enc_start/4 enc_start += line_end*(enc_start/(line_len-line_end)) enc_end = 4*(dec_end/3) enc_end += line_end*(enc_end/(line_len-line_end)) return enc_start, enc_end, dec_skip def pgp_packet_hdr_parse(self, header, prev_partial = False): """ Parse the header of a PGP packet to get the packet type, header length, and data length. Extra trailing characters in header are ignored. prev_partial indicates that the previous packet was a partial packet. An illegal header returns type -1, lengths 0. Header format is defined in RFC4880 section 4. """ hdr = bytearray(header.ljust( 6, chr(0))) if not prev_partial: hdr_len = 1 else: hdr[1:] = hdr # Partial block headers don't have a tag hdr[0] = 0 # Insert a dummy tag. hdr_len = 0 is_partial = False if prev_partial or (hdr[0] & 0xC0) == 0xC0: # New format packet ptag = hdr[0] & 0x3F body_len = hdr[1] lengthtype = 0 hdr_len += 1 if body_len < 192: pass elif body_len <= 223: hdr_len += 1 body_len = ((body_len - 192) << 8) + hdr[2] + 192 elif body_len == 255: hdr_len += 4 body_len = ( (hdr[2] << 24) + (hdr[3] << 16) + (hdr[4] << 8) + hdr[5] ) else: # Partial packet headers are only legal for data packets. if not prev_partial and not ptag in {8,9,11,18}: return (-1, 0, 0, False) # Could do extra testing here. is_partial = True body_len = 1 << (hdr[1] & 0x1F) elif (hdr[0] & 0xC0) == 0x80: # Old format packet ptag = (hdr[0] & 0x3C) >> 2 lengthtype = hdr[0] & 0x03 if lengthtype < 3: hdr_len = 2 body_len = hdr[1] if lengthtype > 0: hdr_len = 3 body_len = (body_len << 8) + hdr[2] if lengthtype > 1: hdr_len = 5 body_len = ( (body_len << 16) + (hdr[3] << 8) + hdr[4] ) else: # Kludgy extra test for compressed packets w/ "unknown" length # gpg generates these in signed-only files. Check for valid # compression algorithm id to minimize false positives. if ptag != 8 or (hdr[1] < 1 or hdr[1] > 3): return (-1, 0, 0, False) hdr_len = 1 body_len = -1 else: return (-1, 0, 0, False) if hdr_len > len(header): return (-1, 0, 0, False) return ptag, hdr_len, body_len, is_partial def sniff(self, data, encoding = None): """ Checks arbitrary data to see if it is a PGP object and returns a set that indicates the kind(s) of object found. The names of the set elements are based on RFC3156 content types with 'pgp-' stripped so they can be used in sniffers for other protocols, e.g. S/MIME. There are additional set elements 'armored' and 'unencrypted'. This code should give no false negatives, but may give false positives. For efficient handling of encoded data, only small segments are decoded. Armored files are detected by their armor header alone. Non-armored data is detected by looking for a sequence of valid PGP packet headers. """ found = set() is_base64 = False is_quopri = False line_len = 0 line_end = 1 enc_start = 0 enc_end = 0 dec_start = 0 skip = 0 ptag = 0 hdr_len = 0 body_len = 0 partial = False offset_enc = 0 offset_dec = 0 offset_packet = 0 # Identify encoding and base64 line length. if encoding and encoding.lower() == 'base64': line_len = data.find('\n') + 1 # Assume uniform length if line_len < 0: line_len = len(data) elif line_len > 1 and data[line_len-2] == '\r': line_end = 2 if line_len - line_end > 76: # Maximum per RFC2045 6.8 return found enc_end = line_len try: segment = base64.b64decode(data[enc_start:enc_end]) except TypeError: return found is_base64 = True elif encoding and encoding.lower() == 'quoted-printable': # Can't selectively decode quopri because encoded length is data # dependent due to escapes! Just decode one medium length segment. # This is enough to contain the first few packets of a long file. try: segment = quopri.decodestring(data[0:1500]) except TypeError: return found # *** ? Docs don't list exceptions is_quopri = True else: line_len = len(data) segment = data # *** Shallow copy? if not segment: found = set() elif not (ord(segment[0]) & 0x80): # Not a PGP packet header if MSbit is 0. Check for armoured data. found.add('armored') if segment.startswith(self.ARMOR_BEGIN_SIGNED): # Clearsigned found.add('unencrypted') found.add('signature') elif segment.startswith(self.ARMOR_BEGIN_SIGNATURE): # Detached signature found.add('signature') elif segment.startswith(self.ARMOR_BEGIN_ENCRYPTED): # PGP uses the same armor header for encrypted and signed only # Fortunately gpg --decrypt handles both! found.add('encrypted') elif segment.startswith(self.ARMOR_BEGIN_PUB_KEY): found.add('key') else: found = set() else: # Could be PGP packet header. Check for sequence of legal headers. while skip < len(segment) and body_len != -1: # Check this packet header. prev_partial = partial ptag, hdr_len, body_len, partial = ( self.pgp_packet_hdr_parse(segment[skip:], prev_partial)) if prev_partial or partial: pass elif ptag == 11: found.add('unencrypted') # Literal Data elif ptag == 1: found.add('encrypted') # Encrypted Session Key elif ptag == 9: found.add('encrypted') # Symmetrically Encrypted Data elif ptag == 18: found.add('encrypted') # Symmetrically Encrypted & MDC elif ptag == 2: found.add('signature') # Signature elif ptag == 4: found.add('signature') # One-Pass Signature elif ptag == 6: found.add('key') # Public Key elif ptag == 14: found.add('key') # Public Subkey elif ptag == 8: # Compressed Data Packet # This is a kludge. Signed, non-encrypted files made by gpg # (but no other gpg files) consist of one compressed data # packet of unknown length which contains the signature # and data packets. # This appears to be an interpretation of RFC4880 2.3. # The compression prevents selective parsing of headers. # So such packets are assumed to be signed messages. if dec_start == 0 and body_len == -1: found.add('signature') found.add('unencrypted') elif ptag < 0 or ptag > 19: found = set() return found dec_start += hdr_len + body_len skip = dec_start if is_base64 and body_len != -1: enc_start, enc_end, skip = self.base64_segment(dec_start, dec_start + 6, 0, line_len, line_end ) segment = base64.b64decode(data[enc_start:enc_end]) if is_base64 and body_len != -1 and skip != len(segment): # End of last packet does not match end of data. found = set() return found def remove_armor(self, text): lines = text.strip().splitlines(True) if lines[0].startswith(self.ARMOR_BEGIN_SIGNED): for idx in reversed(range(0, len(lines))): if lines[idx].startswith(self.ARMOR_BEGIN_SIGNATURE): lines = lines[:idx] while lines and lines[0].strip(): lines.pop(0) break return ''.join(lines).strip() def verify(self, data, signature=None): """ >>> g = GnuPG(None) >>> s = g.sign("Hello, World", _from="smari@mailpile.is", clearsign=True)[1] >>> g.verify(s) """ params = ["--verify"] if signature: sig = tempfile.NamedTemporaryFile() sig.write(signature) sig.flush() params.append(sig.name) params.append("-") self.event.running_gpg(_('Checking signature in %d bytes of data' ) % len(data)) ret, retvals = self.run(params, gpg_input=data, partial_read_ok=True) rp = GnuPGResultParser(debug=self.debug) return rp.parse([None, retvals]).signature_info def encrypt(self, data, tokeys=[], armor=True, sign=False, fromkey=None, throw_keyids=False): """ >>> g = GnuPG(None) >>> g.encrypt("Hello, World", to=["smari@mailpile.is"])[0] 0 """ if tokeys: action = ["--encrypt", "--yes", "--expert", "--trust-model", "always"] for r in tokeys: action.append("--recipient") action.append(r) action.extend([]) self.event.running_gpg(_('Encrypting %d bytes of data to %s' ) % (len(data), ', '.join(tokeys))) else: action = ["--symmetric", "--yes", "--expert"] self.event.running_gpg(_('Encrypting %d bytes of data with password' ) % len(data)) if armor: action.append("--armor") if sign: action.append("--sign") if sign and fromkey: action.append("--local-user") action.append(fromkey) if throw_keyids: action.append("--throw-keyids") if fromkey: self.prepare_passphrase(fromkey, signing=True) retvals = self.run(action, gpg_input=data, send_passphrase=(sign or not tokeys)) return retvals[0], "".join(retvals[1]["stdout"]) def sign(self, data, fromkey=None, armor=True, detach=True, clearsign=False, passphrase=None): """ >>> g = GnuPG(None) >>> g.sign("Hello, World", fromkey="smari@mailpile.is")[0] 0 """ if passphrase is not None: self.passphrase = passphrase.get_reader() if fromkey and passphrase is None: self.prepare_passphrase(fromkey, signing=True) if detach and not clearsign: action = ["--detach-sign"] elif clearsign: action = ["--clearsign"] else: action = ["--sign"] if armor: action.append("--armor") if fromkey: action.append("--local-user") action.append(fromkey) self.event.running_gpg(_('Signing %d bytes of data with %s' ) % (len(data), fromkey or _('default'))) retvals = self.run(action, gpg_input=data, send_passphrase=True) self.passphrase = None return retvals[0], "".join(retvals[1]["stdout"]) def sign_key(self, keyid, signingkey=None): action = ["--yes", "--sign-key", keyid] if signingkey: action.insert(1, "-u") action.insert(2, signingkey) self.event.running_gpg(_('Signing key %s with %s' ) % (keyid, signingkey or _('default'))) retvals = self.run(action, send_passphrase=True) return retvals def delete_key(self, key_fingerprint): cmd = ['--yes', '--delete-secret-and-public-key', key_fingerprint] return self.run(cmd) def recv_key(self, keyid, keyservers=DEFAULT_KEYSERVERS, keyserver_options=(DEFAULT_KEYSERVER_OPTIONS+DEFAULT_IMPORT_OPTIONS)): if not keyid[:2] == '0x': keyid = '0x%s' % keyid self.event.running_gpg(_('Downloading key %s from key servers' ) % (keyid)) for keyserver in keyservers: cmd = ['--keyserver', keyserver, '--recv-key', self._escape_hex_keyid_term(keyid)] for opt in keyserver_options: cmd[2:2] = ['--keyserver-options', opt] retvals = self.run(cmd) if 'unsupported' not in ''.join(retvals[1]["stdout"]): break return self._parse_import(retvals[1]["status"]) def parse_hpk_response(self, lines): results = {} lines = [x.strip().split(":") for x in lines] curpub = None for line in lines: if line[0] == "info": pass elif line[0] == "pub": curpub = line[1] validity = line[6] if line[5]: if int(line[5]) < time.time(): validity += 'e' results[curpub] = { "created": datetime.fromtimestamp(int(line[4])), "created_ts": int(line[4]), "keytype_name": _(openpgp_algorithms.get(int(line[2]), 'Unknown')), "keysize": line[3], "validity": validity, "uids": [], "fingerprint": curpub } elif line[0] == "uid": email, name, comment = parse_uid(urllib.unquote(line[1])) results[curpub]["uids"].append({"name": name, "email": email, "comment": comment}) return results def search_key(self, term, keyservers=DEFAULT_KEYSERVERS, keyserver_options=(DEFAULT_KEYSERVER_OPTIONS+DEFAULT_IMPORT_OPTIONS)): self.event.running_gpg(_('Searching for key for %s in key servers' ) % (term)) for keyserver in keyservers: cmd = ['--keyserver', keyserver, '--fingerprint', '--search-key', self._escape_hex_keyid_term(term)] for opt in keyserver_options: cmd[2:2] = ['--keyserver-options', opt] retvals = self.run(cmd) if 'unsupported' not in ''.join(retvals[1]["stdout"]): break return self.parse_hpk_response(retvals[1]["stdout"]) def get_pubkey(self, keyid): return self.export_pubkeys(selectors=[keyid]) def get_minimal_key(self, key_id=None, user_id=None, armor=True): # Note: We are not stripping revoked subkeys, revocations are # rare but important. A more nuanced approach might only # include *recent* revocations, but we don't have the # tooling for that. args = [ '--export-options', 'export-minimal', '--export-filter', 'drop-subkey=expired-t||disabled-t'] selector = key_id or user_id if not selector: raise ValueError('Export what key?') if user_id: args.extend(['--export-filter', 'keep-uid=uid =~ %s' % user_id]) return self.export_pubkeys( extra_args=args, armor=armor, selectors=[selector]) def export_pubkeys(self, selectors=None, armor=True, extra_args=[]): self.event.running_gpg(_('Exporting keys %s from keychain' ) % (selectors,)) retvals = self.run((extra_args or []) + (['--armor'] if armor else []) + (['--export']) + (selectors or []))[1]["stdout"] return "".join(retvals) def export_privkeys(self, selectors=None): retvals = self.run(['--armor', '--export-secret-keys'] + (selectors or []) )[1]["stdout"] return "".join(retvals) def address_to_keys(self, address): res = {} keys = self.list_keys(selectors=[address]) for key, props in keys.iteritems(): if any([x["email"] == address for x in props["uids"]]): res[key] = props return res def _escape_hex_keyid_term(self, term): """Prepends a 0x to hexadecimal key ids. For example, D13C70DA is converted to 0xD13C70DA. This is required by version 2.x of GnuPG (and is accepted by 1.x). """ is_hex_keyid = False if len(term) == GPG_KEYID_LENGTH or len(term) == 2*GPG_KEYID_LENGTH: hex_digits = set(string.hexdigits) is_hex_keyid = all(c in hex_digits for c in term) if is_hex_keyid: return '0x%s' % term else: return term def chat(self, gpg_args, callback, *args, **kwargs): """This lets a callback have a chat with the GPG process...""" gpg_args = ( self.common_args(interactive=True, will_send_passphrase=True) + [ # We may be interactive, but we're not a human! "--no-tty", "--command-fd=0", "--status-fd=1" ] + (gpg_args or [])) proc = None try: # Here we go! self.debug('Running %s' % ' '.join(gpg_args)) self.event.update_args(gpg_args) proc = Popen(gpg_args, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=0, long_running=True) return callback(proc, *args, **kwargs) finally: # Close this so GPG will terminate. This should already have # been done, but we're handling errors here... if proc and proc.stdin: proc.stdin.close() if proc: self.event.update_return_code(proc.wait()) else: self.event.update_return_code(-1) def GetKeys(gnupg, config, people): keys = [] missing = [] ambig = [] # First, we go to the contact database and get a list of keys. for person in set(people): if '#' in person: keys.append(person.rsplit('#', 1)[1]) else: vcard = config.vcards.get_vcard(person) if vcard: # It is the VCard's job to give us the best key first. lines = [vcl for vcl in vcard.get_all('KEY') if vcl.value.startswith('data:application' '/x-pgp-fingerprint,')] if len(lines) > 0: keys.append(lines[0].value.split(',', 1)[1]) else: missing.append(person) else: missing.append(person) # Load key data from gnupg for use below if keys: all_keys = gnupg.list_keys(selectors=keys) else: all_keys = {} if missing: # Keys are missing, so we try to just search the keychain all_keys.update(gnupg.list_keys(selectors=missing)) found = [] for key_id, key in all_keys.iteritems(): for uid in key.get("uids", []): if uid.get("email", None) in missing: missing.remove(uid["email"]) found.append(uid["email"]) keys.append(key_id) elif uid.get("email", None) in found: ambig.append(uid["email"]) # Next, we go make sure all those keys are really in our keychain. fprints = all_keys.keys() for key in keys: key = key.upper() if key.startswith('0x'): key = key[2:] if key not in fprints: match = [k for k in fprints if k.endswith(key)] if len(match) == 0: missing.append(key) elif len(match) > 1: ambig.append(key) if missing: raise KeyLookupError(_('Keys missing for %s' ) % ', '.join(missing), missing) elif ambig: ambig = list(set(ambig)) raise KeyLookupError(_('Keys ambiguous for %s' ) % ', '.join(ambig), ambig) return keys class OpenPGPMimeSigningWrapper(MimeSigningWrapper): CONTAINER_PARAMS = (('micalg', 'pgp-sha512'), ('protocol', 'application/pgp-signature')) SIGNATURE_TYPE = 'application/pgp-signature' SIGNATURE_DESC = 'OpenPGP Digital Signature' def crypto(self): return GnuPG(self.config, event=self.event) def get_keys(self, who): return GetKeys(self.crypto(), self.config, who) class OpenPGPMimeEncryptingWrapper(MimeEncryptingWrapper): CONTAINER_PARAMS = (('protocol', 'application/pgp-encrypted'), ) ENCRYPTION_TYPE = 'application/pgp-encrypted' ENCRYPTION_VERSION = 1 # FIXME: Define _encrypt, allow throw_keyids def crypto(self): return GnuPG(self.config, event=self.event) def get_keys(self, who): return GetKeys(self.crypto(), self.config, who) class OpenPGPMimeSignEncryptWrapper(OpenPGPMimeEncryptingWrapper): CONTAINER_PARAMS = (('protocol', 'application/pgp-encrypted'), ) ENCRYPTION_TYPE = 'application/pgp-encrypted' ENCRYPTION_VERSION = 1 def crypto(self): return GnuPG(self.config) def _encrypt(self, message_text, tokeys=None, armor=False): from_key = self.get_keys([self.sender])[0] # FIXME: Allow throw_keyids here. return self.crypto().encrypt(message_text, tokeys=tokeys, armor=True, sign=True, fromkey=from_key) def _update_crypto_status(self, part): part.signature_info.part_status = 'verified' part.encryption_info.part_status = 'decrypted' class GnuPGExpectScript(threading.Thread): STARTUP = 'Startup' START_GPG = 'Start GPG' FINISHED = 'Finished' SEND_PASSPHRASE = '!!>STDIN>> [Passphrase from secure passphrase store]') reader = self.sps.get_reader() while True: c = reader.read() if c != '': proc.stdin.write(c) else: proc.stdin.write('\n') break elif line == self.SEND_EOF: self.gnupg.debug('>>STDIN>> [EOF]') proc.stdin.close() elif line is not None: self.gnupg.debug('>>STDIN>> "%s"' % line) proc.stdin.write(line.encode('utf-8')) proc.stdin.write('\n') def _expecter(self, proc, exp, timebox): while timebox[0] > 0: read_char = proc.stdout.read(1) if read_char: self.before += read_char if exp in self.before: if exp: self.gnupg.debug('==Found: %s' % exp) self.before = self.before.split(exp)[0] return True elif read_char == '\n': self.gnupg.debug('< 0) else self.DEFAULT_TIMEOUT timebox = [timeout] self.before = '' try: if not exp: return True self.gnupg.debug('==Expect(%ss): %s' % (timeout, exp)) if RunTimed(timeout, self._expecter, proc, exp, timebox): return True else: raise TimedOut() except TimedOut: timebox[0] = 0 self.gnupg.debug('Timed out') print('Boo! %s not found in %s' % (exp, self.before)) raise def run_script(self, proc, script): for exp, rpl, tmo, state in script: self.expect_exact(proc, exp, timeout=tmo) if rpl: self.sendline(proc, (rpl % self.variables).strip()) if state: self.set_state(state) stderr = proc.stderr.read() if stderr: self.gnupg.debug('<= (2, 1, 12): # Version 2.1.12 was the first version with the key generation # and --pinentry=loopback semantics we require. We just don't # even try with older versions. if variables.get('keytype') == consts.KEYTYPE_CURVE25519: return GnuPG21Curve25519KeyGenerator(gnupg, **kwargs) else: return GnuPG21RSAKeyGenerator(gnupg, **kwargs) else: return GnuPGDummyKeyGenerator(gnupg, **kwargs) # Reset our translation variable _ = gettext ## Include the SKS keyserver certificates here ## KEYSERVER_CERTIFICATE=""" -----BEGIN CERTIFICATE----- MIIFizCCA3OgAwIBAgIJAK9zyLTPn4CPMA0GCSqGSIb3DQEBBQUAMFwxCzAJBgNV BAYTAk5PMQ0wCwYDVQQIDARPc2xvMR4wHAYDVQQKDBVza3Mta2V5c2VydmVycy5u ZXQgQ0ExHjAcBgNVBAMMFXNrcy1rZXlzZXJ2ZXJzLm5ldCBDQTAeFw0xMjEwMDkw MDMzMzdaFw0yMjEwMDcwMDMzMzdaMFwxCzAJBgNVBAYTAk5PMQ0wCwYDVQQIDARP c2xvMR4wHAYDVQQKDBVza3Mta2V5c2VydmVycy5uZXQgQ0ExHjAcBgNVBAMMFXNr cy1rZXlzZXJ2ZXJzLm5ldCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC ggIBANdsWy4PXWNUCkS3L//nrd0GqN3dVwoBGZ6w94Tw2jPDPifegwxQozFXkG6I 6A4TK1CJLXPvfz0UP0aBYyPmTNadDinaB9T4jIwd4rnxl+59GiEmqkN3IfPsv5Jj MkKUmJnvOT0DEVlEaO1UZIwx5WpfprB3mR81/qm4XkAgmYrmgnLXd/pJDAMk7y1F 45b5zWofiD5l677lplcIPRbFhpJ6kDTODXh/XEdtF71EAeaOdEGOvyGDmCO0GWqS FDkMMPTlieLA/0rgFTcz4xwUYj/cD5e0ZBuSkYsYFAU3hd1cGfBue0cPZaQH2HYx Qk4zXD8S3F4690fRhr+tki5gyG6JDR67aKp3BIGLqm7f45WkX1hYp+YXywmEziM4 aSbGYhx8hoFGfq9UcfPEvp2aoc8u5sdqjDslhyUzM1v3m3ZGbhwEOnVjljY6JJLx MxagxnZZSAY424ZZ3t71E/Mn27dm2w+xFRuoy8JEjv1d+BT3eChM5KaNwrj0IO/y u8kFIgWYA1vZ/15qMT+tyJTfyrNVV/7Df7TNeWyNqjJ5rBmt0M6NpHG7CrUSkBy9 p8JhimgjP5r0FlEkgg+lyD+V79H98gQfVgP3pbJICz0SpBQf2F/2tyS4rLm+49rP fcOajiXEuyhpcmzgusAj/1FjrtlynH1r9mnNaX4e+rLWzvU5AgMBAAGjUDBOMB0G A1UdDgQWBBTkwyoJFGfYTVISTpM8E+igjdq28zAfBgNVHSMEGDAWgBTkwyoJFGfY TVISTpM8E+igjdq28zAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4ICAQAR OXnYwu3g1ZjHyley3fZI5aLPsaE17cOImVTehC8DcIphm2HOMR/hYTTL+V0G4P+u gH+6xeRLKSHMHZTtSBIa6GDL03434y9CBuwGvAFCMU2GV8w92/Z7apkAhdLToZA/ X/iWP2jeaVJhxgEcH8uPrnSlqoPBcKC9PrgUzQYfSZJkLmB+3jEa3HKruy1abJP5 gAdQvwvcPpvYRnIzUc9fZODsVmlHVFBCl2dlu/iHh2h4GmL4Da2rRkUMlbVTdioB UYIvMycdOkpH5wJftzw7cpjsudGas0PARDXCFfGyKhwBRFY7Xp7lbjtU5Rz0Gc04 lPrhDf0pFE98Aw4jJRpFeWMjpXUEaG1cq7D641RpgcMfPFvOHY47rvDTS7XJOaUT BwRjmDt896s6vMDcaG/uXJbQjuzmmx3W2Idyh3s5SI0GTHb0IwMKYb4eBUIpQOnB cE77VnCYqKvN1NVYAqhWjXbY7XasZvszCRcOG+W3FqNaHOK/n/0ueb0uijdLan+U f4p1bjbAox8eAOQS/8a3bzkJzdyBNUKGx1BIK2IBL9bn/HravSDOiNRSnZ/R3l9G ZauX0tu7IIDlRCILXSyeazu0aj/vdT3YFQXPcvt5Fkf5wiNTo53f72/jYEJd6qph WrpoKqrwGwTpRUCMhYIUt65hsTxCiJJ5nKe39h46sg== -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR 6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC 9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV /erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z +pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB /wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM 4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV 2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl 0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB NVOFBkpdn627G190 -----END CERTIFICATE----- """ ================================================ FILE: mailpile/crypto/keyinfo.py ================================================ from __future__ import print_function import time import traceback import pgpdump import pgpdump.packet from pgpdump.utils import PgpdumpException, get_int4 from mailpile.util import dict_merge # Patch pgpdump so it stops crashing on weird public keys ##################### def monkey_patch_pgpdump(): # Add Algorithm 22 to the lookup table pgpdump.packet.AlgoLookup.pub_algorithms[22] = 'EdDSA' # Patch the key parser to just silently ignore strange keys orig_pkm = pgpdump.packet.PublicKeyPacket.parse_key_material def _patched_pkm(self, offset): try: return orig_pkm(self, offset) except PgpdumpException: return offset pgpdump.packet.PublicKeyPacket.parse_key_material = _patched_pkm # FIXME: Perhaps we should be checking pgpdump versions? But most of # these are actually API changes, not just bugfixes. It's likely # that versions 1.6+ will continue to throw exceptsions on "unknown" key # types... if/when 1.6 or 2.x get released, we'll just have to revisit # this logic. monkey_patch_pgpdump() # Classes for storing PGP key info ############################################ class ustr(str): def __new__(cls, content): return super(ustr, cls).__new__(cls, content.upper()) class RestrictedDict(dict): KEYS = {} @classmethod def prep_properties(cls): def mk_prop(k): return property(lambda s: s[k], lambda s, v: s.__setitem__(k, v)) for k in cls.KEYS: setattr(cls, k, mk_prop(k)) def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) for k, (t, d) in self.KEYS.items(): if k not in self: if t in (list, dict): self[k] = t() else: self[k] = d def keys(self): kl = list(dict.keys(self)) for dk in (k for k in self.KEYS if k not in kl): kl.append(dk) return sorted(kl) def __setitem__(self, item, value): if item[:1] != '_': if item not in self.KEYS: raise KeyError('Invalid key: %s' % item) if not isinstance(value, self.KEYS[item][0]): try: if isinstance(value, unicode): # Value is unicode, we want other: encode, convert value = self.KEYS[item][0](value.encode('utf-8')) elif isinstance(value, str): # Value is not unicode, we want unicode: decode, convert value = self.KEYS[item][0](value.decode('utf-8')) else: # Neither unicode nor string, just try to convert value = self.KEYS[item][0](value) except (TypeError, ValueError): raise TypeError( 'Bad type for %s: %s (want %s)' % (item, value, self.KEYS[item][0].__name__)) dict.__setitem__(self, item, value) def __getitem__(self, item): if item[:1] == '_': return dict.__getitem__(self, item) else: return dict.get(self, item, self.KEYS[item][1]) class KeyUID(RestrictedDict): KEYS = { 'name': (unicode, ''), 'email': (str, ''), 'comment': (unicode, '')} def __repr__(self): parts = [] if self['name']: parts.append(self['name']) if self['email']: parts.append('<%s>' % self['email']) if self['comment']: parts.append('(%s)' % self['comment']) return ' '.join(parts) class KeyInfo(RestrictedDict): KEY_TRUSTED_CODES = ('u', 'f') # Note: Ignoring marginal keys KEY_INVALID_CODES = ('i', 'd', 'e', 'r', 'n') KEYS = { 'fingerprint': (ustr, 'MISSING'), 'capabilities': (str, ''), 'keytype_name': (str, 'unknown'), 'keytype_code': (int, 0), 'keysize': (int, 0), 'created': (int, 0), 'expires': (int, 0), 'validity': (str, '?'), 'key_source': (str, None), 'uids': (list, None), 'subkeys': (list, None), 'is_subkey': (bool, False), 'have_secret': (bool, False), 'on_keychain': (bool, False), 'in_vcards': (bool, False)} expired = property(lambda k: time.time() > k.expires > 0) is_usable = property(lambda k: (k.validity not in k.KEY_INVALID_CODES and not k.expired)) can_encrypt = property(lambda k: ('e' in k.capabilities.lower() and k.is_usable)) can_sign = property(lambda k: ('s' in k.capabilities.lower() and k.is_usable)) def summary(self, full_fingerprint=False): """ Generate a short string summarizing the key's main properties: key ID, UIDs, expiration date, algorithm, size, capabilities, and validity. Note: If summary ends with !, the key is invalid/unusable. """ now = time.time() emails = ','.join(sorted([u.email for u in self.uids if u.email])) return '%s%s%s/%s%s/%s%s' % ( self.fingerprint[-(9999 if full_fingerprint else 16):], ('=%s' % emails) if emails else '', ('<%x' % self.expires) if self.expires else '', self.keytype_name[:3], self.keysize, self.capabilities, ('' if self.is_usable else '!')) def __repr__(self): if self.is_subkey: return self.summary() return '{ %s }' % '\n '.join( '%-12s = %s' % (k, self[k]) for k in self.keys() if self[k] is not None) def ensure_autocrypt_uid(keyinfo, ac_uid): """Ensure we include the email from the Autocrypt header in a UID.""" if keyinfo.is_subkey: return found = 0 for uid in keyinfo.uids: if uid.email == ac_uid.email: uid.comment = uid.comment + '(Autocrypt)' found += 1 if not found: keyinfo.uids += [ac_uid] def add_subkey_capabilities(keyinfo, now=None): """Make key "inherit" the capabilities of any un-expired subkeys.""" now = now or time.time() key_caps = set(c for c in keyinfo.capabilities if c in ('c', 'e', 's')) combined_caps = set(c.upper() for c in key_caps) for subkey in keyinfo.subkeys: if not (0 < subkey.expires < now): combined_caps |= set(c.upper() for c in subkey.capabilities) keyinfo.capabilities = '%s%s' % ( ''.join(sorted(list(combined_caps))), ''.join(sorted(list(key_caps)))) def synthesize_validity(keyinfo, now=None): """Synthesize key validity property.""" # FIXME: Revocations? now = now or time.time() if (0 < keyinfo.expires < now and keyinfo.validity not in keyinfo.KEY_INVALID_CODES): keyinfo.validity = 'e' def recalculate_expiration(keyinfo, now=None): """Adjust the main expiration date to take subkeys into account.""" now = now or time.time() # For each capability, figure out what is the latest expiration date # provided by a subkey for that capability. expirations = {} for cap in set(c for c in keyinfo.capabilities if c in ('C', 'E', 'S')): for subkey in keyinfo.subkeys: if subkey.expires and not (0 < subkey.expires < now): expirations[cap] = max(subkey.expires, expirations.get(cap, 0)) for cap in expirations: # If the subkey is not expired, and provides a capability our # main key doesn't have, then its expiration date matters. if cap.lower() not in keyinfo.capabilities: keyinfo.expires = min(subkey.expires, keyinfo.expires) @classmethod def FromGPGI(cls, gpgi_keyinfo): mki = cls( created=int(gpgi_keyinfo.get("creation_date_ts", gpgi_keyinfo.get('created_ts', 0))), expires=int(gpgi_keyinfo.get("expiration_date_ts", 0)), capabilities=gpgi_keyinfo.get("capabilities", ""), have_secret=gpgi_keyinfo.get("secret", False)) for k in ('fingerprint', 'validity', 'keytype_name'): mki[k] = str(gpgi_keyinfo[k]) for k in ('keysize', ): mki[k] = int(gpgi_keyinfo[k]) for uid in gpgi_keyinfo.get('uids', []): mki.uids.append(KeyUID( name=uid.get("name", ""), email=uid.get("email", ""), comment=uid.get("comment", ""))) mki.capabilities = ''.join(sorted([c for c in mki.capabilities])) return mki class MailpileKeyInfo(KeyInfo): KEYS = dict_merge(KeyInfo.KEYS, { 'vcards': (dict, None), 'origins': (list, None), 'is_autocrypt': (bool, False), 'is_gossip': (bool, False), 'is_preferred': (bool, False), 'is_pinned': (bool, False), 'scores': (dict, None), 'score_stars': (int, 0), 'score_reason': (unicode, None), 'score': (int, 0)}) KeyUID.prep_properties() KeyInfo.prep_properties() MailpileKeyInfo.prep_properties() def get_keyinfo(data, autocrypt_header=None, key_info_class=KeyInfo, key_uid_class=KeyUID, key_source=None): """ This method will parse a stream of OpenPGP packets into a list of KeyInfo objects. Note: Signatures are not validated, this code only parses the data. """ try: if "-----BEGIN" in data: ak = pgpdump.AsciiData(data) else: ak = pgpdump.BinaryData(data) packets = list(ak.packets()) except (TypeError, IndexError, PgpdumpException): traceback.print_exc() return [] def _unixtime(packet, seconds=0, days=0): return (packet.raw_creation_time + (days or 0) * 24 * 3600 + (seconds or 0)) results = [] last_uid = key_uid_class() # Dummy last_key = key_info_class() # Dummy last_pubkeypacket = None main_key_id = None for m in packets: try: if isinstance(m, pgpdump.packet.PublicKeyPacket): size = str(int(1.024 * round(len('%x' % (m.modulus or 0)) / 0.256))) last_pubkeypacket = m last_key = key_info_class( key_source=key_source, fingerprint=m.fingerprint, keytype_name=m.pub_algorithm or '', keytype_code=m.raw_pub_algorithm, keysize=size) if isinstance(m, pgpdump.packet.PublicSubkeyPacket): last_key.is_subkey = True results[-1].subkeys.append(last_key) else: main_key_id = m.key_id results.append(last_key) # Older pgpdumps may fail here and cause traceback noise, but # the loop will limp onwards. last_key.created = _unixtime(m) if m.raw_days_valid > 0: last_key.expires = _unixtime(m, days=m.raw_days_valid) if last_key.expires == last_key.created: last_key.expires = 0 elif isinstance(m, pgpdump.packet.UserIDPacket) and results: last_uid = key_uid_class(name=m.user_name, email=m.user_email) last_key.uids.append(last_uid) elif isinstance(m, pgpdump.packet.SignaturePacket) and results: # Note: We don't actually check the signature; we trust # GnuPG will if we decide to use this key. if m.key_id == main_key_id: for s in m.subpackets: if s.subtype == 9: exp = _unixtime(last_pubkeypacket, seconds=get_int4(s.data, 0)) last_key.expires = max(last_key.expires, exp) elif s.subtype == 27: caps = set(c for c in last_key.capabilities) for flag, c in ((0x01, 'c'), (0x02, 's'), (0x0C, 'e'), (0x20, 'a')): if s.data[0] & flag: caps.add(c) last_key.capabilities = ''.join(caps) except (TypeError, AttributeError, KeyError, IndexError, NameError): traceback.print_exc() autocrypt_uid = None if autocrypt_header: # The autocrypt spec tells us that the visible addr= attribute # overrides whatever is on the key itself, so we synthesize a # fake UID here so the info is correct in an Autocrypt context. autocrypt_uid = key_uid_class( email=autocrypt_header['addr'], comment='Autocrypt') now = time.time() for keyinfo in results: keyinfo.synthesize_validity(now=now) keyinfo.add_subkey_capabilities(now=now) keyinfo.recalculate_expiration(now=now) if autocrypt_uid is not None: keyinfo.ensure_autocrypt_uid(autocrypt_uid) return results if __name__ == "__main__": import sys for f in sys.argv[1:]: with open(f, 'r') as fd: keyinfo = get_keyinfo(fd.read())[0] print('%s' % keyinfo) print('%s' % keyinfo.summary(full_fingerprint=True)) print('Is usable = %s, Can encrypt = %s, Can sign = %s' % ( keyinfo.is_usable, keyinfo.can_encrypt, keyinfo.can_sign)) print('') # EOF ================================================ FILE: mailpile/crypto/mime.py ================================================ from __future__ import print_function # These are methods to do with MIME and crypto, implementing PGP/MIME. import math import random import re import StringIO import email.parser from email import encoders from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from mailpile.crypto.state import EncryptionInfo, SignatureInfo from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailutils.generator import Generator ##[ Common utilities ]######################################################### def Normalize(payload): # http://tools.ietf.org/html/rfc3156 says we must: # # - use CRLF everywhere # - strip trailing whitespace # - end with a CRLF # # In particlar, the stripping of trailing whitespace seems (based on # experiments with mutt), to in practice mean "strip trailing whitespace # off the last line"... # text = re.sub(r'\r?\n', '\r\n', payload).rstrip(' \t') if not text.endswith('\r\n'): text += '\r\n' return text def MessageAsString(part, unixfrom=False): buf = StringIO.StringIO() Generator(buf).flatten(part, unixfrom=unixfrom) return Normalize(buf.getvalue()).replace('--\r\n--', '--\r\n\r\n--') class EncryptionFailureError(ValueError): def __init__(self, message, to_keys): ValueError.__init__(self, message) self.to_keys = to_keys class SignatureFailureError(ValueError): def __init__(self, message, from_key): ValueError.__init__(self, message) self.from_key = from_key DEFAULT_CHARSETS = ['utf-8', 'iso-8859-1'] def _decode_text_part(part, payload, charsets=None): for cs in (c for c in ([part.get_content_charset() or None] + (charsets or DEFAULT_CHARSETS)) if c): try: return cs, payload.decode(cs) except (UnicodeDecodeError, TypeError, LookupError): pass return '8bit', payload def _update_text_payload(part, payload, charsets=None): if 'content-transfer-encoding' in part: # We want this recalculated by the charset setting below del part['content-transfer-encoding'] charset, payload = _decode_text_part(part, payload, charsets=charsets) part.set_payload(payload, charset) ##[ Methods for unwrapping encrypted parts ]################################### def MimeAttachmentDisposition(part, kind, newpart): """ Create a Content-Disposition header for a processed attachment using the original file name if available from the PGP packet, otherwise try stripping the extension from the unprocessed attachment file name. """ # Delete embedded \n and \r (shouldn't get_filename() do this itself??). filename = part.get_filename().replace('\n','').replace('\r','') if filename: filename = filename.decode('utf-8', 'replace') part.encryption_info["description"] = _("Decrypted: %s") % filename if part.encryption_info.filename: newfilename = part.encryption_info.filename else: # get_filename() can parse quoted, folded and RFC2231 names. # If there's no filename in Content-Disposition it tries Content-Type. newfilename = filename if 'armored' in kind and newfilename.endswith('.asc'): newfilename = newfilename[:len(newfilename)-len('.asc')] elif newfilename.endswith('.gpg'): newfilename = newfilename[:len(newfilename)-len('.gpg')] # add_header() does quoting, folding, maybe someday RFC2231?. newpart.add_header('Content-Disposition', 'attachment', filename=newfilename.encode('utf-8')) def MimeReplacePart(part, newpart, keep_old_headers=False): """ Replace a MIME part with new version (decrypted, signature verified, ... ). retaining headers from the old part that are not in the new part. The headers that would be overwritten will be renamed and kept if the keep_old_headers variable is set to a prefix string. MIME headers (Content-*) get special treatment. Returns a set of the headers that got copied from the new part. """ part.set_payload(newpart.get_payload()) # Original MIME headers must go, whether we're replacing them or not. for hdr in [k for k in part.keys() if k.lower().startswith('content-')]: while hdr in part: del part[hdr] # If we're keeping the non-MIME old headers, make copies now before # they get deleted below. if keep_old_headers: if not isinstance(keep_old_headers, str): keep_old_headers = "Old" for h in newpart.keys(): headers = (part.get_all(h) or []) if (len(headers) == 1) and (part[h] == newpart[h]): continue for v in headers: part.add_header('X-%s-%s' % (keep_old_headers, h), v) for h in newpart.keys(): while h in part: del part[h] copied = set([]) for h, v in newpart.items(): part.add_header(h, v) if not h.lower().startswith('content-'): copied.add(h) return copied def UnwrapMimeCrypto(part, protocols=None, psi=None, pei=None, charsets=None, unwrap_attachments=True, require_MDC=True, depth=0, sibling=0, efail_unsafe=False, allow_decrypt=True): """ This method will replace encrypted and signed parts with their contents and set part attributes describing the security properties instead. """ # Guard against maliciously constructed emails if depth > 6: return part.signature_info = SignatureInfo(parent=psi) part.encryption_info = EncryptionInfo(parent=pei) part.signed_headers = set([]) part.encrypted_headers = set([]) mimetype = part.get_content_type() or 'text/plain' disposition = part['content-disposition'] or "" encoding = part['content-transfer-encoding'] or "" # FIXME: Check the protocol. PGP? Something else? # FIXME: This is where we add hooks for other MIME encryption # schemes, so route to callbacks by protocol. crypto_cls = protocols['openpgp'] if part.is_multipart(): # Containers are by default not bubbly part.signature_info.bubbly = False part.encryption_info.bubbly = False if part.is_multipart() and mimetype == 'multipart/signed': try: boundary = part.get_boundary() payload, signature = part.get_payload() # The Python get_payload() method likes to rewrite headers, # which breaks signature verification. So we manually parse # out the raw payload here. head, raw_payload, junk = part.as_string( ).replace('\r\n', '\n').split('\n--%s\n' % boundary, 2) part.signature_info = crypto_cls().verify( Normalize(raw_payload), signature.get_payload()) part.signature_info.bubble_up(psi) # Reparent the contents up, removing the signature wrapper hdrs = MimeReplacePart(part, payload, keep_old_headers='PH-Renamed') part.signed_headers = hdrs # Try again, in case we just unwrapped another layer # of multipart/something. UnwrapMimeCrypto(part, protocols=protocols, psi=part.signature_info, pei=part.encryption_info, charsets=charsets, unwrap_attachments=unwrap_attachments, require_MDC=require_MDC, depth=depth+1, sibling=sibling, efail_unsafe=efail_unsafe, allow_decrypt=allow_decrypt) except (IOError, OSError, ValueError, IndexError, KeyError): part.signature_info = SignatureInfo() part.signature_info["status"] = "error" part.signature_info.bubble_up(psi) elif part.is_multipart() and mimetype == 'multipart/encrypted': try: if not allow_decrypt: raise ValueError('Decryption forbidden, MIME structure is weird') preamble, payload = part.get_payload() (part.signature_info, part.encryption_info, decrypted) = ( crypto_cls().decrypt( payload.as_string(), require_MDC=require_MDC)) except (IOError, OSError, ValueError, IndexError, KeyError): part.encryption_info = EncryptionInfo() part.encryption_info["status"] = "error" part.signature_info.bubble_up(psi) part.encryption_info.bubble_up(pei) if part.encryption_info['status'] == 'decrypted': newpart = email.parser.Parser().parsestr(decrypted) # Reparent the contents up, removing the encryption wrapper hdrs = MimeReplacePart(part, newpart, keep_old_headers='MH-Renamed') # Is there a Memory-Hole/Protected-Headers force-display part? pl = part.get_payload() if hdrs and isinstance(pl, list): if (pl[0]['content-type'].startswith('text/rfc822-headers;') and 'protected-headers' in pl[0]['content-type']): # Parse these headers as well and override the top level, # again. This is to be sure we see the same thing as # everyone else (same algo as enigmail). data = email.parser.Parser().parsestr( pl[0].get_payload(), headersonly=True) for h in data.keys(): while h in part: del part[h] part[h] = data[h] hdrs.add(h) # Finally just delete the part, we're done with it! del pl[0] part.encrypted_headers = hdrs if part.signature_info["status"] != 'none': part.signed_headers = hdrs # Try again, in case we just unwrapped another layer # of multipart/something. UnwrapMimeCrypto(part, protocols=protocols, psi=part.signature_info, pei=part.encryption_info, charsets=charsets, unwrap_attachments=unwrap_attachments, require_MDC=require_MDC, depth=depth+1, sibling=sibling, efail_unsafe=efail_unsafe, allow_decrypt=allow_decrypt) # If we are still multipart after the above shenanigans (perhaps due # to an error state), recurse into our subparts and unwrap them too. elif part.is_multipart(): for count, sp in enumerate(part.get_payload()): # EFail mitigation: We decrypt attachments and the first part # of a nested multipart structure, but not any subsequent parts. # This allows rewriting of messages to *append* cleartext, but # disallows rewriting that pushes "inline" encrypted content # further down to where the recipient might not notice it. sp_disp = (unwrap_attachments and sp['content-disposition']) or "" allow_decrypt = (efail_unsafe or (count == sibling == 0) or sp_disp.startswith('attachment')) and allow_decrypt UnwrapMimeCrypto(sp, protocols=protocols, psi=part.signature_info, pei=part.encryption_info, charsets=charsets, unwrap_attachments=unwrap_attachments, require_MDC=require_MDC, depth=depth+1, sibling=count, efail_unsafe=efail_unsafe, allow_decrypt=allow_decrypt) elif disposition.startswith('attachment'): # The sender can attach signed/encrypted/key files without following # rules for naming or mime type. # So - sniff to detect parts that need processing and identify protocol. kind = '' for protocol in protocols: crypto_cls = protocols[protocol] kind = crypto_cls().sniff(part.get_payload(), encoding) if kind: break if unwrap_attachments and ('encrypted' in kind or 'signature' in kind): # Messy! The PGP decrypt operation is also needed for files which # are encrypted and signed, and files that are signed only. payload = part.get_payload( None, True ) try: if not allow_decrypt: raise ValueError('Decryption forbidden, MIME structure is weird') (part.signature_info, part.encryption_info, decrypted) = ( crypto_cls().decrypt( payload, require_MDC=require_MDC)) except (IOError, OSError, ValueError, IndexError, KeyError): part.encryption_info = EncryptionInfo() part.encryption_info["status"] = "error" part.signature_info.bubble_up(psi) part.encryption_info.bubble_up(pei) if (part.encryption_info['status'] == 'decrypted' or part.signature_info['status'] == 'verified'): # Force base64 encoding and application/octet-stream type newpart = MIMEBase('application', 'octet-stream') newpart.set_payload(decrypted) encoders.encode_base64(newpart) # Add Content-Disposition with appropriate filename. MimeAttachmentDisposition(part, kind, newpart) MimeReplacePart(part, newpart) # Is there another layer to unwrap? UnwrapMimeCrypto(part, protocols=protocols, psi=part.signature_info, pei=part.encryption_info, charsets=charsets, unwrap_attachments=unwrap_attachments, require_MDC=require_MDC, depth=depth+1, sibling=sibling, efail_unsafe=efail_unsafe, allow_decrypt=allow_decrypt) else: # FIXME: Best action for unsuccessful attachment processing? pass elif mimetype == 'text/plain': return UnwrapPlainTextCrypto(part, protocols=protocols, psi=psi, pei=pei, charsets=charsets, require_MDC=require_MDC, depth=depth+1, sibling=sibling, efail_unsafe=efail_unsafe, allow_decrypt=allow_decrypt) else: # FIXME: This is where we would handle cryptoschemes that don't # appear as multipart/... pass # Mix in our bubbles part.signature_info.mix_bubbles() part.encryption_info.mix_bubbles() # Bubble up! part.signature_info.bubble_up(psi) part.encryption_info.bubble_up(pei) def UnwrapPlainTextCrypto(part, protocols=None, psi=None, pei=None, charsets=None, require_MDC=True, depth=0, sibling=0, efail_unsafe=False, allow_decrypt=True): """ This method will replace encrypted and signed parts with their contents and set part attributes describing the security properties instead. """ payload = part.get_payload(None, True).strip() si = SignatureInfo(parent=psi) ei = EncryptionInfo(parent=pei) for crypto_cls in protocols.values(): crypto = crypto_cls() if (payload.startswith(crypto.ARMOR_BEGIN_ENCRYPTED) and payload.endswith(crypto.ARMOR_END_ENCRYPTED)): try: if not allow_decrypt: raise ValueError('Decryption forbidden, MIME structure is weird') si, ei, text = crypto.decrypt(payload, require_MDC=require_MDC) _update_text_payload(part, text, charsets=charsets) except (IOError, OSError, ValueError, IndexError, KeyError): ei = EncryptionInfo() ei["status"] = "error" break elif (payload.startswith(crypto.ARMOR_BEGIN_SIGNED) and payload.endswith(crypto.ARMOR_END_SIGNED)): try: si = crypto.verify(payload) except (IOError, OSError, ValueError, IndexError, KeyError): si = SignatureInfo() si["status"] = "error" _update_text_payload(part, crypto.remove_armor(payload), charsets=charsets) break part.signature_info = si part.signature_info.bubble_up(psi) part.encryption_info = ei part.encryption_info.bubble_up(pei) ##[ Methods for stripping down message headers ]############################### def ObscureSubject(subject): """ Replace the Subject line with something nondescript. """ return '...' def ObscureNames(hdr): """ Remove names (leaving e-mail addresses) from the To: and Cc: headers. >>> ObscureNames("Bjarni R. E. , e@b.c (Elmer Boop)") u', ' """ from mailpile.mailutils.addresses import AddressHeaderParser return ', '.join('<%s>' % ai.address for ai in AddressHeaderParser(hdr)) def ObscureSender(sender): """ Remove as much metadata from the From: line as possible. """ return ObscureNames(sender) def ObscureAllRecipients(sender): """ Remove all content from the To: and Cc: lines entirely. """ return "recipients-suppressed;" # A dictionary for use with MimeWrapper's obscured_headers parameter, # that will obscure only what is required from the public header. OBSCURE_HEADERS_REQUIRED = { 'autocrypt-gossip': lambda t: None} # A dictionary for use with MimeWrapper's obscured_headers parameter, # that will obscure as much of the metadata from the public header as # possible without breaking compatibility. OBSCURE_HEADERS_MILD = { 'subject': ObscureSubject, 'from': ObscureSender, 'sender': ObscureSender, 'reply-to': ObscureSender, 'to': ObscureNames, 'cc': ObscureNames, 'user-agent': lambda t: None, 'autocrypt-gossip': lambda t: None} # A dictionary for use with MimeWrapper's obscured_headers parameter, # that will obscure as much of the metadata from the public header as # possible. This is only useful with encrypted messages and will badly # break things unless the recipient is running an MUA that fully implements # Memory Hole / Protected Headers. OBSCURE_HEADERS_EXTREME = { 'subject': ObscureSubject, 'from': ObscureSender, 'sender': ObscureSender, 'reply-to': ObscureSender, 'to': ObscureAllRecipients, 'cc': lambda t: None, 'date': lambda t: None, 'in-reply-to': lambda t: None, 'references': lambda t: None, 'openpgp': lambda t: None, 'user-agent': lambda t: None, 'autocrypt-gossip': lambda t: None} ##[ Methods for encrypting and signing ]####################################### class MimeWrapper: CONTAINER_TYPE = 'multipart/mixed' CONTAINER_PARAMS = () # These are the default "memory hole" settings; wrap/protect the # important user-visible headers. WRAPPED_HEADERS = ('subject', 'from', 'to', 'cc', 'date', 'user-agent', 'sender', 'reply-to', 'in-reply-to', 'references', 'openpgp') # Force-displayed headers; if these headers get obscured, add a # visible part that shows them to the user in legacy clients. FORCE_DISPLAY_HEADERS = ('subject', 'from', 'to', 'cc') # By default, no headers are obscured. That's a user preference, # since there's a trade-off between privacy and compatibility. OBSCURED_HEADERS = OBSCURE_HEADERS_REQUIRED def __init__(self, config, event=None, cleaner=None, sender=None, recipients=None, use_html_wrapper=False, wrapped_headers=None, obscured_headers=None): from mailpile.mailutils.emails import MakeBoundary self.config = config self.event = event self.sender = sender self.cleaner = cleaner self.recipients = recipients or [] self.use_html_wrapper = use_html_wrapper self.container = c = MIMEMultipart(boundary=MakeBoundary()) self.wrapped_headers = self.WRAPPED_HEADERS if wrapped_headers is not None: self.wrapped_headers = wrapped_headers or () self.obscured_headers = self.OBSCURED_HEADERS if obscured_headers is not None: self.obscured_headers = obscured_headers or {} c.set_type(self.CONTAINER_TYPE) c.signature_info = SignatureInfo(bubbly=False) c.encryption_info = EncryptionInfo(bubbly=False) if self.cleaner: self.cleaner(self.container) for pn, pv in self.CONTAINER_PARAMS: self.container.set_param(pn, pv) def crypto(self): return NotImplementedError("Please override me") def attach(self, part): c = self.container c.attach(part) if not hasattr(part, 'signature_info'): part.signature_info = SignatureInfo(parent=c.signature_info) part.encryption_info = EncryptionInfo(parent=c.encryption_info) else: part.signature_info.parent = c.signature_info part.signature_info.bubbly = True part.encryption_info.parent = c.encryption_info part.encryption_info.bubbly = True if self.cleaner: self.cleaner(part) del part['MIME-Version'] return self def get_keys(self, people): return people def flatten(self, msg, unixfrom=False): return MessageAsString(msg, unixfrom=unixfrom) def get_only_text_part(self, msg): count = 0 only_text_part = None for part in msg.walk(): if part.is_multipart(): continue count += 1 mimetype = part.get_content_type() or 'text/plain' if mimetype != 'text/plain' or count != 1: return False else: only_text_part = part return only_text_part def wrap(self, msg, **kwargs): # Subclasses override return msg def prepare_wrap(self, msg): obscured = self.obscured_headers wrapped = self.wrapped_headers obscured_set = set([]) to_delete = {} for (h, header_value) in msg.items(): if not header_value: continue hl = h.lower() if hl == 'mime-version': to_delete[h] = True elif not hl.startswith('content-'): if hl in obscured: obscured_set.add(h) oh = obscured[hl](header_value) if oh: self.container.add_header(h, oh) else: self.container.add_header(h, header_value) if hl not in wrapped and hl not in obscured: to_delete[h] = True for h in to_delete: while h in msg: del msg[h] if hasattr(msg, 'signature_info'): self.container.signature_info = msg.signature_info self.container.encryption_info = msg.encryption_info return self.force_display_headers(msg, obscured_set) def force_display_headers(self, msg, obscured_set): # If we aren't changing the structure of the message (adding a # force-display part), we can just wrap the original and be done. if not [k for k in obscured_set if k.lower() in self.FORCE_DISPLAY_HEADERS]: return msg header_display = MIMEBase('text', 'rfc822-headers', protected_headers="v1") header_display['Content-Disposition'] = 'inline' container = MIMEBase('multipart', 'mixed') container.attach(header_display) container.attach(msg) # Cleanup... for p in (msg, header_display, container): if 'MIME-Version' in p: del p['MIME-Version'] if self.cleaner: self.cleaner(header_display) self.cleaner(msg) # NOTE: The copying happens at the end here, because we need the # cleaner (on msg) to have run first. display_headers = [] to_delete = {} for h, v in msg.items(): hl = h.lower() if not hl.startswith('content-') and not hl.startswith('mime-'): container.add_header(h, v) if hl in self.FORCE_DISPLAY_HEADERS and h in obscured_set: display_headers.append('%s: %s' % (h, v)) to_delete[h] = True for h in to_delete: while h in msg: del msg[h] header_display.set_payload('\r\n'.join(reversed(display_headers))) return container class MimeSigningWrapper(MimeWrapper): CONTAINER_TYPE = 'multipart/signed' CONTAINER_PARAMS = () SIGNATURE_TYPE = 'application/x-signature' SIGNATURE_DESC = 'Abstract Digital Signature' def __init__(self, *args, **kwargs): MimeWrapper.__init__(self, *args, **kwargs) name = ('OpenPGP-digital-signature.html' if self.use_html_wrapper else 'OpenPGP-digital-signature.asc') self.sigblock = MIMEBase(*self.SIGNATURE_TYPE.split('/')) self.sigblock.set_param("name", name) for h, v in (("Content-Description", self.SIGNATURE_DESC), ("Content-Disposition", "attachment; filename=\"%s\"" % name)): self.sigblock.add_header(h, v) def _wrap_sig_in_html(self, sig): return ( "

%(title)s

\n\n%(description)s\n\n

" "
\n%(sig)s\n
" "%(ad)s." ) % self._wrap_sig_in_html_vars(sig) def _wrap_sig_in_html_vars(self, sig): return { # FIXME: We deliberately do not flag these messages for i18n # translation, since we rely on 7-bit content here so as # not to complicate the MIME structure of the message. "title": "Digital Signature", "description": ( "This is a digital signature, which can be used to verify\n" "the authenticity of this message. You can safely discard\n" "or ignore this file if your e-mail software does not\n" "support digital signatures."), "ad": "Generated by Mailpile", "ad_url": "https://www.mailpile.is/", # FIXME: Link to help? "sig": sig} def _update_crypto_status(self, part): part.signature_info.part_status = 'verified' def wrap(self, msg, prefer_inline=False): from_key = self.get_keys([self.sender])[0] if prefer_inline: prefer_inline = self.get_only_text_part(msg) else: prefer_inline = False if prefer_inline is not False: message_text = Normalize(prefer_inline.get_payload(None, True) .strip() + '\r\n\r\n') status, sig = self.crypto().sign(message_text, fromkey=from_key, clearsign=True, armor=True) if status == 0: _update_text_payload(prefer_inline, sig) self._update_crypto_status(prefer_inline) return msg else: msg = self.prepare_wrap(msg) self.attach(msg) self.attach(self.sigblock) message_text = self.flatten(msg) status, sig = self.crypto().sign(message_text, fromkey=from_key, armor=True) if status == 0: if self.use_html_wrapper: sig = self._wrap_sig_in_html(sig) self.sigblock.set_payload(sig) self._update_crypto_status(self.container) return self.container raise SignatureFailureError(_('Failed to sign message!'), from_key) class MimeEncryptingWrapper(MimeWrapper): CONTAINER_TYPE = 'multipart/encrypted' CONTAINER_PARAMS = () ENCRYPTION_TYPE = 'application/x-encrypted' ENCRYPTION_VERSION = 0 def __init__(self, *args, **kwargs): MimeWrapper.__init__(self, *args, **kwargs) self.version = MIMEBase(*self.ENCRYPTION_TYPE.split('/')) self.version.set_payload('Version: %s\n' % self.ENCRYPTION_VERSION) for h, v in (("Content-Disposition", "attachment"), ): self.version.add_header(h, v) self.enc_data = MIMEBase('application', 'octet-stream') for h, v in (("Content-Disposition", "attachment; filename=\"OpenPGP-encrypted-message.asc\""), ): self.enc_data.add_header(h, v) self.attach(self.version) self.attach(self.enc_data) def _encrypt(self, message_text, tokeys=None, armor=False): return self.crypto().encrypt(message_text, tokeys=tokeys, armor=True) def _update_crypto_status(self, part): part.encryption_info.part_status = 'decrypted' def _add_padding(self, text, chunksize=None): """Add up to 16kB of whitespace to the end of a message as padding.""" if chunksize is None: chunksize = min(max(160, 2 ** int(math.log(len(text), 2))), 8192) pad = ('\r\n' + (' ' * 78)) * int(chunksize / 80) return (text + (pad if random.randint(0, 1) else '') + (pad[:chunksize - (len(text) % chunksize)])) def wrap(self, msg, prefer_inline=False): to_keys = set(self.get_keys(self.recipients + [self.sender])) if prefer_inline: prefer_inline = self.get_only_text_part(msg) else: prefer_inline = False if prefer_inline is not False: message_text = self._add_padding( Normalize(prefer_inline.get_payload(None, True)), # This padding is user facing, so don't overdo it. chunksize=160) status, enc = self._encrypt(message_text, tokeys=to_keys, armor=True) if status == 0: _update_text_payload(prefer_inline, enc) self._update_crypto_status(prefer_inline) return msg else: msg = self.prepare_wrap(msg) if self.cleaner: self.cleaner(msg) message_text = self._add_padding(self.flatten(msg)) status, enc = self._encrypt(message_text, tokeys=to_keys, armor=True) if status == 0: self.enc_data.set_payload(enc) self._update_crypto_status(self.enc_data) return self.container raise EncryptionFailureError(_('Failed to encrypt message!'), to_keys) if __name__ == "__main__": import sys import doctest # FIXME: Add tests for the wrapping/unwrapping code. It's crazy that # we don't have such tests. :-( results = doctest.testmod(optionflags=doctest.ELLIPSIS) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/crypto/records.py ================================================ # # NOTE: THIS CODE IS NOT BEING USED - YET. # IT IS HERE TO FACILITATE REVIEW, COMMENTS AND EXPERIMENTS. # # FIXME: Respond to following comment from Kai Michaelis # # The dictionaries in records.py completely lack any message # authentication and thus are vulnerable. The cryptographic checksums # of the plaintext do not prevent this. The comments mention that this # is intentional to limit destruction in case of bit flips. This is # misguided in two ways. First, it's the responsibility of the file # system and transport protocol to prevent these and second, # cryptographic checksums are not designed for error detecting or # correction. # ... # To derive keys for multiple shards in a EncryptedDictionary use a # proper KDF like HKDF [2] or a CPRNG like those defined in NIST # SP800-90a rev1 [3] (except Dual_EC DRBG :wink:). # # Full discussion here: https://github.com/mailpile/Mailpile/pull/1684 # # FIXME: We've switched from AES-CBC to AES-CTR. We should review the # IV logic and make sure it is still sane given the different # properties of the underlying algorithms. # """ Record-based AES encrypted data storage This is a collection of modules designed to allow for easy Pythonic use of encrypted data stores. The basics are provided by EncryptedRecordStore, which defines the on-disk storage format (largely compatible with RFC2822) and takes care of the encryption and decryption of records. The EncryptedBlobStore provides a subset of Pythonic list semantics, extending EncryptedRecordStore to allow for arbitrarily sized elements and splitting the storage accross multiple files to play nice with backups, network-based storage and other environments where huge files might be a problem. The EncryptedDict provides a subset of Pythonic dict semantics, using a mixture of the above two classes. Notes: 1. A simple SHA256 digest is used to derive an AES key. This would not be considered sufficient for low entropy (human generated) keys. 2. All data storage is record based, which implies nontrivial amounts of disk space may be wasted. On the other hand, this is good for security as it obscures the size of the data being stored. 3. The EncryptedDict uses the encryption key as a salt to ensure the hashing is not predictable to an attacker. Again, low entropy keys should be avoided. 4. No provisions are made to make it possible to change keys. 5. No buffering or caching of any kind is done. 6. Deleting entries is NOT supported anywhere. Overwriting works. Performance thoughts: The key to performance of these algorithms will ultimately depend on how well the OS caches data. In general we can help with that by encouraging hot spots, trying to cluster frequently used values together. The other way we can help is to minimize wasted space within the records themselves, so the OS doesn't waste RAM caching junk. For the metadata index, we expect hot spot clustering to focus around recently received mail. The default sorts are by date and most of the time users are reading or organizing recently received mail. So the OS should have a relatively easy time effectively caching records for current metadata. The case for posting lists, which naturally live in an EncryptedDict is different but also promising. In general, the keyword index will grow linearly with the number of index messages; in particular each message ID is unique and will generate one or more new keywords in the index. Thus the keyword index will have a very long tail of rarely used, small entries. The expected performance of such entries (reads and writes) is dominated by the disk seeks times. For these entries, we can save at least one disk seek by storing the values along with the keys, but that is about all we can do. Conversely, some keywords will have a very high frequency; for example virtually all English language messages will contain the word "the". These common keywords will become hot spots during the indexing process, so causing them to cluster together will again let us play nice with the operating system caches. A basic strategy for this is to allow larger entries to bump smaller ones from the front of each hash bucket to later stages, as entry size correlates with keyword popularity. Another optimization which is out of scope for this module, is they should compress well as they will contain long sequential runs of message IDs. From a frequency point of view, the middle-of-the-road keywords barely matter; 94% of all keywords match fewer than 5 messages, about 0.16% match over 1000 messages. On average about 17 keywords are generated per message. Detailed search keyword stats: emails keywords __________ratios__________ 2 4551468 88.21% 100.00% 88.21% 4 290686 5.63% 11.79% 93.84% 8 150924 2.92% 6.16% 96.77% 16 74239 1.44% 3.23% 98.21% 32 38341 0.74% 1.79% 98.95% 64 20552 0.40% 1.05% 99.35% 128 13178 0.26% 0.65% 99.60% 256 7731 0.15% 0.40% 99.75% 512 4640 0.09% 0.25% 99.84% 1024 2793 0.05% 0.16% 99.90% 2048 2141 0.04% 0.10% 99.94% >2048 3084 0.06% 0.06% 100.00% (sample size: ~300k emails) """ from __future__ import print_function from __future__ import absolute_import import binascii import hashlib import os import struct import time import threading from .aes_utils import getrandbits, aes_ctr_encrypt, aes_ctr_decrypt class _SimpleList(object): """Some syntactic sugar for listalikes""" def append(self, value): with self._lock: pos = len(self) self[pos] = value return pos def extend(self, values): for v in values: self.append(v) def __getslice__(self, i, j): return [self[v] for v in range(i, j)] def __iadd__(self, y): self.extend(y) def __iter__(self): return (self[v] for v in range(0, len(self))) def __reversed__(self): return (self[v] for v in reversed(range(0, len(self)))) class EncryptedRecordStore(_SimpleList): """ This is an on-disk AES encrypted record storage. Data is written out base64 encoded, with 64 characters per line + CRLF, as that is likely to make it in and out of IMAP servers or other mail stores unchanged, while giving us a round multiple of what both AES and Base64 can handle without padding. """ _HEADER = ('X-Mailpile-Encrypted-Records: v1\r\n' 'From: Mailpile \r\n' 'Subject: %(filename)s\r\n' 'cipher: aes-128-ctr\r\n' 'record-size: %(record_size)d\r\n' 'iv-seed: %(ivs)s\r\n' '\r\n') def __init__(self, fn, key, max_bytes=400, overwrite=False, default_value=None): key_hash = bytes(hashlib.sha256(key.encode('utf-8')).digest()) self._NEWLINE = '\r\n' self._max_bytes = max_bytes self._calculate_constants() self._default_value = default_value self._iv_seed = getrandbits(48) self._iv_seed_random = '%x' % getrandbits(64) self._aes_key = key_hash[:16] self._header_data = { 'record_size': self._RECORD_SIZE, 'filename': fn, 'ivs': '%12.12x' % self._iv_seed } self._lock = threading.RLock() try: self._fd = open(fn, 'wb+' if overwrite else 'rb+') except (OSError, IOError): self._fd = open(fn, 'wb+') if not self._parse_header(): self._write_header() if max_bytes > self._max_bytes: raise AssertionError('max_bytes mismatch') def _calculate_constants(self): # Calculate our constants! Magic numbers: # - 48 is how much data fits in 64 byte of base64 # - 16 is the size of the IV # - 7 is the minimum size of our checksum self._RECORD_LINES = max(1, int((self._max_bytes + 7 + 16) / 48) + 1) self._RECORD_SIZE = self._RECORD_LINES * 48 self._MAX_DATA_SIZE = self._RECORD_SIZE - (7 + 16) self._RECORD_LINE_BYTES = 64 self._ZERO_RECORD = '\0' * self._RECORD_SIZE self._RECORD_BYTES = self._RECORD_LINES * (self._RECORD_LINE_BYTES + len(self._NEWLINE)) def _parse_header(self): try: with self._lock: self._fd.seek(0) header = self._fd.read(len(self._HEADER % self._header_data) + 1024) if not header: return False if '\n' in header and self._NEWLINE not in header: self._NEWLINE = '\n' header = header.split(self._NEWLINE + self._NEWLINE)[0] headers = dict(hl.strip().split(': ', 1) for hl in header.splitlines()) self._header_skip = len(header) + len(self._NEWLINE) * 2 self._iv_seed = long(headers['iv-seed'], 16) + 10240020 self._max_bytes = int(headers['record-size']) - (7 + 16) - 1 self._header_data = { 'record_size': int(headers['record-size']), 'filename': headers['Subject'], 'ivs': headers['iv-seed'] } self._calculate_constants() return True except IOError: return False except (KeyError, ValueError, AssertionError): import traceback traceback.print_exc() return False def _write_header(self): with self._lock: self._header_data['ivs'] = '%12.12x' % self._iv_seed header = self._HEADER % self._header_data self._fd.seek(0) self._fd.write(header) self._fd.flush() self._header_skip = len(header) def close(self): self._write_header() self._fd.close() def _iv(self, pos): # Here we generate an IV that should never repeat: the first 6 bytes # are derived from a random counter created at program startup, the # rest is pseudorandom crap. with self._lock: self._iv_seed += 1 self._iv_seed %= 0x1000000000000 if (self._iv_seed % 123456) == 0: self._write_header() self._iv_seed_random = hashlib.sha512(self._iv_seed_random ).digest() iv = bytes(struct.pack(' self._MAX_DATA_SIZE: raise ValueError('Data too big for record') pos = int(pos) if pos < 0: raise KeyError('Negative record position') iv = self._iv(pos) # We're using MD5 as a checksum here to detect corruption. cks = hashlib.md5(data).hexdigest() # We use the checksum as padding, where we are guaranteed by the # assertion above to have room for at least 6 nybbles of the MD5. record = (data + ':' + cks + self._ZERO_RECORD )[:self._RECORD_SIZE - len(iv)] encrypted = (iv + aes_ctr_encrypt(self._aes_key, iv, record) ).encode('base64').replace('\n', '') if len(encrypted) != self._RECORD_LINES * self._RECORD_LINE_BYTES: raise AssertionError('%d: <%s> %s != %d*%d' % (pos, encrypted, len(encrypted), self._RECORD_LINES, self._RECORD_LINE_BYTES)) with self._lock: self._fd.seek(self._header_skip + pos * self._RECORD_BYTES) for i in range(0, self._RECORD_LINES): self._fd.write(encrypted[i * self._RECORD_LINE_BYTES : (i+1) * self._RECORD_LINE_BYTES]) self._fd.write(self._NEWLINE) def save_zeros(self, pos, count): zrecord = (('A' * self._RECORD_LINE_BYTES) + self._NEWLINE ) * self._RECORD_LINES with self._lock: self._fd.seek(self._header_skip + pos * self._RECORD_BYTES) for i in range(0, count): self._fd.write(zrecord) def load_record(self, pos): pos = int(pos) if pos < 0: raise KeyError('Negative record position') with self._lock: self._fd.seek(self._header_skip + pos * self._RECORD_BYTES) try: encrypted = self._fd.read(self._RECORD_BYTES).decode('base64') except (IOError, binascii.Error): encrypted = '' if encrypted == '': if self._default_value is not None: return self._default_value raise KeyError('Failed to read at %s' % pos) if (len(encrypted) != self._RECORD_SIZE): raise KeyError('Incorrect record size %s at %s' % (len(encrypted), pos)) iv, encrypted = encrypted[:16], encrypted[16:] if iv == ('\x00' * 16) and self._default_value is not None: return self._default_value plaintext, checksum = aes_ctr_decrypt(self._aes_key, iv, encrypted ).rsplit(':', 1) checksum = bytes(checksum.replace('\0', '')) if (len(checksum) < 6): raise ValueError('Checksum too short') if not hashlib.md5(plaintext).hexdigest().startswith(checksum): raise ValueError('Checksum mismatch') return plaintext def __getitem__(self, pos): try: return self.load_record(pos) except ValueError: raise KeyError(pos) def __setitem__(self, pos, data): return self.save_record(pos, data) def get(self, pos, default=None): try: return self[pos] except KeyError: return default def __len__(self): with self._lock: self._fd.seek(0, 2) size = self._fd.tell() return (size - self._header_skip) // self._RECORD_BYTES class EncryptedBlobStore(_SimpleList): """ This is an on-disk variable-sized, sharded AES encrypted blob storage. It augments EncryptedRecordStore by placing bounds on how large each encrypted file will become and allows data blobs of arbitrary size by bumping "large" records to a secondary (or tertiary, ...) store. """ _BIG_POINTER = '\0B->' def __init__(self, base_fn, key, max_bytes=400, shard_size=50000, big_ratio=10, overwrite=False): self._base_fn = base_fn self._key = key self._max_bytes = max(max_bytes, 50) self._shard_size = max(int(shard_size), 1024) self._big_ratio = float(big_ratio) self._overwrite = overwrite self._lock = threading.RLock() self._shards = [] self._load_next_shard() while os.path.exists(self._next_shardname()): self._load_next_shard() self._big_map = {} self._big_shard = None s0 = property(lambda self: self._shards[0]) def _big(self): with self._lock: if self._big_shard is None: self._big_shard = EncryptedBlobStore( '%s-b' % self._base_fn, self._key, max_bytes=self._max_bytes * self._big_ratio, shard_size=self._shard_size / self._big_ratio, big_ratio=self._big_ratio, overwrite=self._overwrite) return self._big_shard def _next_shardname(self): return '%s-%s' % (self._base_fn, len(self._shards) + 1) def _load_next_shard(self): with self._lock: self._shards.append(EncryptedRecordStore( self._next_shardname(), self._key, max_bytes=self._max_bytes, overwrite=self._overwrite)) def __getitem__(self, pos): shard, pos = pos // self._shard_size, pos % self._shard_size value = self._shards[shard][pos] if value.startswith(self._BIG_POINTER): self._big_map[pos] = int(value[len(self._BIG_POINTER):], 16) return self._big()[self._big_map[pos]] else: return value def __setitem__(self, pos, data): shard, pos = pos // self._shard_size, pos % self._shard_size with self._lock: while shard >= len(self._shards): self._load_next_shard() if len(data) > self.s0._MAX_DATA_SIZE: with self._lock: bpos = self._big_map.get(pos, len(self._big())) self._big()[bpos] = data self._big_map[pos] = bpos data = '%s%x' % (self._BIG_POINTER, bpos) self._shards[shard][pos] = data def __len__(self): with self._lock: return (self._shard_size * (len(self._shards) - 1) + len(self._shards[-1])) def close(self): with self._lock: for s in self._shards: s.close() if self._big_shard is not None: self._big_shard.close() self._shards = [] self._big_shard = None class EncryptedUnicodeStore(EncryptedBlobStore): """ An EncryptedBlobStore that only works with unicode data. """ def __setitem__(self, pos, data): return EncryptedBlobStore.__setitem__(self, pos, data.encode('utf-8')) def __getitem__(self, pos): return EncryptedBlobStore.__getitem__(self, pos).decode('utf-8') class EncryptedDict(object): """ This is a variable-sized, sharded AES encrypted dict. It uses EncryptedRecordStore for hashing and EncryptedBlobStore for values that do not fit alongside the keys. At the moment, data can be overwritten, but not deleted. TODO: - Grow the dict by adding keysets - Migrating "exciting" keys to the primary keyset """ DEFAULT_BUCKET_SIZE = 5 DEFAULT_DIGEST_SIZE = 16 # 128 bit hashes _UNUSED = '\0U' _DELETED = '\0D' MIN_KEY_BYTES = (48 - 16 - 7 - 1) # Magic number, smallest possible # key record size. DEFAULT_KEY_BYTES = (48 - 16 - 7 - 1) + 1*48 # 2 lines per record DEFAULT_DATA_BYTES = (48 - 16 - 7 - 1) + 8*48 # 9 lines ber record def __init__(self, base_fn, key, key_bytes=None, data_bytes=None, big_ratio=5, shard_size=100000, min_shards=1, shard_ratio=2.0, bucket_size=None, digest_size=None, overwrite=False, init_zeros=True, sparse=False): self._base_fn = base_fn self._key = key self._key_bytes = key_bytes or self.DEFAULT_KEY_BYTES self._data_bytes = (self.DEFAULT_DATA_BYTES if (data_bytes is None) else data_bytes) self._shard_ratio = shard_ratio # Growth ratio when expanding self._shard_size = max(int(shard_size), 1024) self._big_ratio = float(big_ratio) self._bucket_size = bucket_size or self.DEFAULT_BUCKET_SIZE self._digest_size = digest_size or self.DEFAULT_DIGEST_SIZE self._overwrite = overwrite self._sparse = sparse self._init_zeros = init_zeros self._lock = threading.RLock() self.load_factor = [] self.writes = [] self.reads = [] self._keys = [] while (os.path.exists(self._next_keyfile()[0]) or len(self._keys) < min_shards): self._load_next_keys() self.reset_counters() if big_ratio > 0: self._values = EncryptedBlobStore(self._base_fn, self._key, max_bytes=data_bytes, shard_size=shard_size, big_ratio=big_ratio, overwrite=overwrite) else: self._values = None def reset_counters(self): with self._lock: self.load_factor = [1.0 for keyset in self._keys] self.writes = [0 for keyset in self._keys] self.reads = [0 for keyset in self._keys] def _keyfile_size(self, kfi): return int(self._shard_size * (self._shard_ratio**kfi)) def _keyfile_bytes(self, kfi): return self._key_bytes def _next_keyfile(self): pos = len(self._keys) return '%s-k-%s' % (self._base_fn, pos + 1), pos def _load_next_keys(self): with self._lock: kf, kfi = self._next_keyfile() kb = self._keyfile_bytes(kfi) ow = self._overwrite or not os.path.exists(kf) self._keys.append(EncryptedRecordStore(kf, self._key, max_bytes=kb, overwrite=ow, default_value=self._UNUSED)) self.load_factor.append(0) self.writes.append(0) self.reads.append(0) if ow: kfs = self._keyfile_size(kfi) if self._sparse: self._keys[kfi][kfs - 1] = self._UNUSED elif self._init_zeros: self._keys[kfi].save_zeros(0, kfs) else: for i in range(0, kfs): self._keys[kfi][i] = self._UNUSED return kfi, self._keys[kfi] def _digest(self, key): digest = hashlib.sha256(self._key) if isinstance(key, unicode): key = key.encode('utf-8') elif not isinstance(key, str): key = str(key) digest.update(key) return digest.digest()[:self._digest_size] def _offset(self, digest): return struct.unpack(''): vpos = rdata[self._digest_size + 1:] vpos = struct.unpack('', vpos, len(value))]) else: keyset[rpos] = ''.join([ digest, struct.pack('', self._values.append(value), len(value))]) else: keyset[rpos] = record_data return rpos elif on_fail is not None: return on_fail(self, kfi, keyset, pos, digest, value, records) else: return None def save_digest_record(self, digest, value, on_fail=None): pos = self._offset(digest) for kfi, keyset in enumerate(self._keys): rpos = self._try_save(kfi, keyset, pos, digest, value, on_fail=on_fail) if rpos is not None: self.writes[kfi] += 1 return keyset, rpos # If we get this far, then we need to grow... if self._shard_ratio > 0: kfi, keyset = self._load_next_keys() rpos = self._try_save(kfi, keyset, pos, digest, value, on_fail=on_fail) if rpos is not None: self.writes[kfi] += 1 return keyset, rpos raise KeyError('Save failed at %s' % pos) def save_record(self, key, value, on_fail=None): return self.save_digest_record(self._digest(key), value, on_fail=on_fail) def __setitem__(self, key, value): self.save_record(key, value) def __contains__(self, key): try: keyset, (rpos, rdata) = self.load_record(key) return True except (KeyError, ValueError, IndexError): return False def __delitem__(self, key): try: keyset, (rpos, rdata) = self.load_record(key) keyset[rpos] = self._DELETED except (KeyError, ValueError): pass def values(self): for keyset in self._keys: for val in keyset: try: yield self.rdata_value(val) except (KeyError, ValueError, IndexError): pass def rdata_digest(self, rdata): return rdata[:self._digest_size] def rdata_value(self, rdata): if rdata[self._digest_size] == '=': return rdata[self._digest_size + 1:] elif rdata[self._digest_size] == '>': pos, dlen = struct.unpack(' %s' % (i, ed.get(str(i)))) done = time.time() print ('10k dict reads in %.2f (%.8f s/op)\n -- reads=%s lf=%s' % (done - t0, (done - t0) / count, ed.reads, ed.load_factor)) ================================================ FILE: mailpile/crypto/state.py ================================================ from __future__ import print_function #. Common crypto state and structure import copy from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n class KeyLookupError(ValueError): def __init__(self, message, missing): ValueError.__init__(self, message) self.missing = missing # Crypto state is a strange beast, it needs to flow down and then # back up again - parts need to inherit a crypto context from their # container, but if something interesting is discovered that has to # be overridden. # # The discoveries then usually have to bubble up to influence the # overall state of that container and the message itself (mixed-* # states, etc). # class CryptoInfo(dict): """Base class for crypto-info classes""" KEYS = ["protocol", "status", "description"] STATUSES = ["none", "mixed-error", "error"] DEFAULTS = {"status": "none"} def __init__(self, parent=None, copy=None, bubbly=True): self.parent = parent self.bubbly = bubbly self.filename = None self.bubbles = [] self._status = None if copy: self.update(copy) self._status = self.get("status") elif parent: self.update(parent) self._status = self.get("status") else: self.update(self.DEFAULTS) part_status = property(lambda self: (self._status or self.DEFAULTS["status"]), lambda self, v: self._set_status(v)) def _set_status(self, value): if value not in self.STATUSES: print('Bogus status for %s: %s' % (type(self), value)) raise ValueError('Invalid status: %s' % value) self._status = value self.mix_bubbles() def __setitem__(self, item, value): if item not in self.KEYS: raise KeyError('Invalid key: %s' % item) if item == "status": if value not in self.STATUSES: print('Bogus status for %s: %s' % (type(self), value)) raise ValueError('Invalid value for %s: %s' % (key, value)) if self._status is None: # Capture initial value self._status = value dict.__setitem__(self, item, value) def _overwrite_with(self, ci): for k in self.keys(): del self[k] self.update(ci) def bubble_up(self, parent=None): # Bubbling up adds this context to the list of contexts to be # evaluated for all parent states. if parent is not None and parent != self: self.parent = parent # Some contexts are neutral (pure MIME boilerplate) and do not # bubble up at all. if not self.bubbly: return parent = self.parent while parent is not None: parent.bubbles.append(self) parent = parent.parent def mix_bubbles(self): # Reset visible status to initial state self["status"] = self.part_status # Mix in all the bubbly bubbles for bubble in self.bubbles: if bubble.bubbly: self._mix_in(bubble) def _mix_in(self, ci): """ This generates a mixed state for the message. The most exciting state is returned/explained, the status prefixed with "mixed-". How exciting states are, is determined by the order of the STATUSES attribute. This is lossy, but hopefully in a useful and non-harmful way. """ status = self["status"] if self.STATUSES.index(status) <= self.STATUSES.index(ci.part_status): # ci is MORE or EQUALLY interesting mix = copy.copy(ci) if (self.bubbly and status != mix.part_status and not mix.part_status.startswith('mixed-')): mix["status"] = "mixed-%s" % mix.part_status else: mix["status"] = mix.part_status self._overwrite_with(mix) elif not status.startswith("mixed-"): # ci is LESS interesting self["status"] = 'mixed-%s' % status class EncryptionInfo(CryptoInfo): """Contains information about the encryption status of a MIME part""" KEYS = (CryptoInfo.KEYS + ["have_keys", "missing_keys", "locked_keys"]) STATUSES = (CryptoInfo.STATUSES + ["mixed-decrypted", "decrypted", "mixed-missingkey", "missingkey", "mixed-lockedkey", "lockedkey"]) class SignatureInfo(CryptoInfo): """Contains information about the signature status of a MIME part""" KEYS = (CryptoInfo.KEYS + ["name", "email", "keyinfo", "timestamp"]) STATUSES = (CryptoInfo.STATUSES + ["mixed-unknown", "unknown", "mixed-changed", "changed", # TOFU; not the key we expected! "mixed-unsigned", "unsigned", # TOFU; should be signed! "mixed-expired", "expired", "mixed-revoked", "revoked", "mixed-unverified", "unverified", "mixed-signed", "signed", # TOFU; signature matches history "mixed-verified", "verified", "mixed-invalid", "invalid"]) ================================================ FILE: mailpile/crypto/streamer.py ================================================ from __future__ import print_function # # This is code to stream data to or from encrypted storage. If the invoking # code us correctly written, it should be able to work with data far in # excess of available RAM. # # The storage format "Mailpile Encrypted Storage" takes pains to be either # valid RFC2822 (for direct storage in IMAP servers) or a delimited format # similar to OpenPGP armour. Files must use one style or the other, not a # mixture of both. # # By default this code prefers to use AES-128-CTR. This cipher is malleable, # which means that data corruption is localized and does not affect the # rest of the file (unlike CBC, for example). In order to detect corruption # or attacks, a SHA256-based MAC can be calculated on the plaintext or # the ciphertext (or both). When verifying the plaintext, the sum is written # into the header of the file, when verifying the ciphertext the sum is # expected to be stored somewhere else. # ############################################################################## # FIXME: # # The decryption routines here support "MEP v1" which used AES-256-CBC # and MD5 sums (not a proper MAC). # # Very few people have data in this format, it would be nice to just # delete all of that code once users have had time to migrate. But # for that to happen there needs to be a migration that finds old data # and re-encrypts it automatically. We don't have that yet! # ############################################################################## # import base64 import os import hashlib import sys import re import threading import time import traceback from datetime import datetime from tempfile import NamedTemporaryFile import mailpile.platforms from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.safe_popen import Popen, PIPE from mailpile.util import CryptoLock, safe_remove, safe_assert from mailpile.util import sha512b64 as genkey from mailpile.crypto.aes_utils import getrandbits from mailpile.crypto.aes_utils import aes_ctr_encryptor, aes_ctr_decryptor PREFERRED_CIPHER = 'aes-128-ctr' # This is for backwards compatibility with v1 of our storage format; we have # since corrected our silliness and v2 uses a SHA256-based MAC. LEN_MD5_SUM = len(hashlib.md5('testing').hexdigest()) MD5_SUM_FORMAT = 'md5sum: %s' MD5_SUM_PLACEHOLDER = MD5_SUM_FORMAT % ('0' * LEN_MD5_SUM) MD5_SUM_RE = re.compile('(?m)^' + MD5_SUM_FORMAT % (r'[^\n]+',)) LEN_SHA_256 = len(hashlib.sha256('testing').hexdigest()) SHA_256_FORMAT = 'sha256: %s' SHA_256_PLACEHOLDER = SHA_256_FORMAT % ('0' * LEN_SHA_256) SHA_256_RE = re.compile('(?m)^' + SHA_256_FORMAT % (r'[^\n]+',)) BLANK_LINE_RE = re.compile('^\s*$') # This gets populated with all the obsolete data we see during # decryption, the app can check for this to trigger migrations. PREFERRED_FORMAT = 'v2:%s' % PREFERRED_CIPHER DETECTED_OBSOLETE_FORMATS = set([]) OPENSSL_COMMAND = mailpile.platforms.GetDefaultOpenSSLCommand OPENSSL_MD_ALG = "md5" # FIXME: Why does Windows require this? Move to mailpile.platforms when # we understand the underlying issue. if sys.platform.startswith("win"): FILTER_MD5 = True else: FILTER_MD5 = False def mac_sha256(key, data): mac = hashlib.sha256(key or '') mac.update(data or '') return mac.hexdigest() class IOFilter(threading.Thread): """ This class will wrap a filehandle and spawn a background thread to filter either the input or output. """ BLOCKSIZE = 16 * 1024 def __init__(self, fd, callback, name=None, error_callback=None, blocksize=None): threading.Thread.__init__(self) self.callback = callback self.error_callback = error_callback self.exc_info = None self.blocksize = blocksize or self.BLOCKSIZE self.fd = fd self.writing = None self.reading_from = None self.writing_to = None self.exposed_fd = None self.my_pipe_fd = None pipe = os.pipe() self.pipe_reader = os.fdopen(pipe[0], 'rb', 0) self.pipe_writer = os.fdopen(pipe[1], 'wb', 0) self.info = 'Starting' self.aborting = False if name: self.name = name def __str__(self): return '%s: %s' % (threading.Thread.__str__(self), self.info) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def close(self): fd, self.exposed_fd = self.exposed_fd, None if fd is not None: try: fd.close() except OSError: # May already have been closed, that's fine pass if self.writing is False: self.aborting = 'Closed reader' self.join() def join(self, aborting=None): if aborting is not None: self.aborting = aborting return threading.Thread.join(self) def writer(self): if self.writing is None: self.writing = True self.reading_from = self.pipe_reader self.writing_to = self.fd self.my_pipe_fd = self.pipe_reader self.exposed_fd = self.pipe_writer self.start() return self.pipe_writer def reader(self): if self.writing is None: self.daemon = True self.writing = False self.reading_from = self.fd self.writing_to = self.pipe_writer self.my_pipe_fd = self.pipe_writer self.exposed_fd = self.pipe_reader self.start() return self.pipe_reader def _copy_loop(self): while not self.aborting: self.info = 'reading' data = self.reading_from.read(self.blocksize) if not self.aborting: if len(data) == 0: self.info = 'writing, EOF' self.writing_to.write(self.callback(None) or '') break else: self.info = 'writing' self.writing_to.write(self.callback(data)) def run(self): okay = [AssertionError] if self.writing is False: # If we close early, we may get ValueErrors okay.append(ValueError) try: self.info = 'Starting: %s' % self.writing self._copy_loop() self.info += ', done' except tuple(okay): self.info += ', okay' pass except: self.info += ', failed' self.exc_info = sys.exc_info() traceback.print_exc() if self.error_callback: try: self.error_callback() except: pass finally: fd, self.my_pipe_fd = self.my_pipe_fd, None if fd is not None: self.info = 'Closing' fd.close() self.info = 'Dead' class ReadLineIOFilter(IOFilter): """ This is a line-based IOFilter, which can stop when it sees a particular marker to hand off processing to others. """ def __init__(self, fd, callback, start_data=None, stop_check=None, **kwargs): self.stop_check = stop_check self.buffered = list(start_data) self.buf_bytes = sum(len(s) for s in start_data) IOFilter.__init__(self, fd, callback, **kwargs) def _maybe_flush(self, eof=False): if eof or (self.buf_bytes >= self.blocksize): i, self.info = self.info, 'flushing' self.writing_to.write(self.callback(''.join(self.buffered))) self.buffered = [] self.buf_bytes = 0 while eof: self.info = 'flushing EOF' data = self.callback(None) or '' self.writing_to.write(data) if not data: break self.info = i def _copy_loop(self): self.info = 'copying' for line in self.reading_from: if self.aborting: return self.buffered.append(line) if not re.match(BLANK_LINE_RE, line): # Don't count blank lines self.buf_bytes += len(line) self._maybe_flush() if self.aborting or (self.stop_check and self.stop_check(line)): break if not self.aborting: self._maybe_flush(eof=True) class IOCoprocess(object): def __init__(self, command, fd, name=None, long_running=False): self.stderr = '' self._retval = None self._reading = False self.command = command self.name = name if command: try: self._proc, self._fd = self._popen(command, fd, long_running) except: print('Popen(%s, %s, %s)' % (command, fd, long_running)) traceback.print_exc() print() raise else: self._proc, self._fd = None, fd def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def close(self, *args): if self._retval is None: proc, fd, self._proc, self._fd = self._proc, self._fd, None, None if proc and fd: fd.close(*args) self.stderr = proc.stderr.read() # If we were reading from the process, not writing, then # closing our FD above may not be enough to terminate it, # and the following calls may hang. So kill kill kill. if self._reading: count = 0 while proc.poll() is None: time.sleep(0.01) if count == 4: print('TERM => %s' % proc) proc.terminate() elif count > 9: print('KILL => %s' % proc) proc.kill() break count += 1 self._retval = proc.wait() else: self._retval = 0 return self._retval class OutputCoprocess(IOCoprocess): """ This class will stream data to an external coprocess. """ def _popen(self, command, fd, long_running): proc = Popen(command, stdin=PIPE, stdout=fd, stderr=PIPE, bufsize=0, long_running=long_running) return proc, proc.stdin def _write_filter(self, data): return data def write(self, data, *args, **kwargs): return self._fd.write(self._write_filter(data), *args, **kwargs) def flush(self): return self._fd.flush() class InputCoprocess(IOCoprocess): """ This class will stream data from an external coprocess. """ def _popen(self, command, fd, long_running): self._reading = True proc = Popen(command, stdin=fd, stdout=PIPE, stderr=PIPE, bufsize=0, long_running=long_running) return proc, proc.stdout def _read_filter(self, data): return data def __iter__(self, *args): return (self._read_filter(ln) for ln in self._fd.__iter__(*args)) def readline(self, *args): return self._read_filter(self._fd.readline(*args)) def readlines(self, *args): return [self._read_filter(ln) for ln in self._fd.readlines(*args)] def read(self, *args): return self._read_filter(self._fd.read(*args)) class ChecksummingStreamer(OutputCoprocess): """ This checksums and streams data to a named temporary file on disk, which can then be read back or linked to a final location. """ FILTER_BLOCKSIZE = None def __init__(self, dir=None, name=None, long_running=False, use_filter=FILTER_MD5): self.tempfile, self.temppath = self._mk_tempfile_and_path(dir) self.name = name self.outer_sha256 = None if use_filter: self.outer_sha = hashlib.sha256() self.shafilter = IOFilter(self.tempfile, self._sha256_callback, name='%s/sha256' % (self.name or 'css'), blocksize=self.FILTER_BLOCKSIZE) self.fd = self.shafilter.writer() else: self.outer_sha = None self.fd = self.tempfile self.saved = False self.finished = False try: self._write_preamble() OutputCoprocess.__init__(self, self._mk_command(), self.fd, name=self.name, long_running=long_running) except: try: self.fd.close() if self.outer_sha is not None: self.shafilter.close() self.tempfile.close() safe_remove(self.temppath) except (IOError, OSError): pass raise def _mk_tempfile_and_path(self, _dir): ntf = NamedTemporaryFile(dir=_dir, delete=False) return ntf, ntf.name def _mk_command(self): return None def finish(self): fin, self.finished = self.finished, True if fin: return # Stop sending output to our coprocess, wait for it to finish OutputCoprocess.close(self) # Write postamble (the shafilter), close that too self.tempfile.seek(0, 2) self._write_postamble() if self.outer_sha is None: # If we weren't doing the SHA256 on the fly, do it now. self.calculate_outer_sha256() else: # Otherwise, close our coprocess to trigger the calculation self.fd.close() self.shafilter.close() # Reset our tempfile to the beginning for reading self.tempfile.seek(0, 0) def close(self): self.finish() self.tempfile.close() def save(self, filename, finish=True, mode='wb'): if finish: self.finish() # If no filename, return contents to caller if filename is None: if not self.saved: safe_remove(self.temppath) self.saved = True self.tempfile.seek(0, 0) return self.tempfile.read() # 1st save just renames the tempfile exists = os.path.exists(filename) if (not self.saved and (('a' not in mode) or not exists)): try: if exists: os.remove(filename) os.rename(self.temppath, filename) self.saved = True return except (OSError, IOError): pass # 2nd save (or append to existing) creates a copy with open(filename, mode) as out: self.save_copy(out) if not self.saved: safe_remove(self.temppath) self.saved = True def calculate_outer_sha256(self): self.tempfile.seek(0, 0) data = self.tempfile.read(4096) self.outer_sha = outer_sha = hashlib.sha256() while data != '': # We calculate the MD5 sum as if the data used the CRLF linefeed # convention, whether it's actually using that or not. outer_sha.update(data.replace('\r', '').replace('\n', '\r\n')) data = self.tempfile.read(4096) self.outer_sha256 = outer_sha.hexdigest() def outer_mac_sha256(self): # Hm, we have no key, so this is a bit pointless return mac_sha256('', self.outer_sha.digest()) def save_copy(self, ofd): self.tempfile.seek(0, 0) data = self.tempfile.read(4096) while data != '': ofd.write(data) data = self.tempfile.read(4096) def _sha256_callback(self, data): if data is None: # EOF... self.outer_sha256 = self.outer_sha.hexdigest() return '' else: # We calculate the MD5 sum as if the data used the CRLF linefeed # convention, whether it's actually using that or not. self.outer_sha.update(data.replace('\r', '').replace('\n', '\r\n')) return data def _write_preamble(self): pass def _write_postamble(self): pass class EncryptingDelimitedStreamer(ChecksummingStreamer): """ This class creates a coprocess for encrypting data. The data will be streamed to a named temporary file on disk, which can then be read back or linked to a final location. """ BEGIN_DATA = "-----BEGIN MAILPILE ENCRYPTED DATA-----\n" EXTRA_HEADERS = "X-Mailpile-Encrypted-Data: v2\n" EXTRA_DATA = {} END_DATA = "-----END MAILPILE ENCRYPTED DATA-----\n" PREFERRED_CIPHER = None FILTER_BLOCKSIZE = 19 * 3 * 16 # Make AES and Base64 happy def __init__(self, key, dir=None, cipher=None, name=None, header_data=None, long_running=False, use_filter=FILTER_MD5): self.cipher = cipher or self.PREFERRED_CIPHER or PREFERRED_CIPHER self.nonce, self.key = self._nonce_and_mutated_key(key) self.header_data = (header_data if header_data is not None else self.EXTRA_DATA) self.encode_buffer = '' if self.cipher == 'aes-128-ctr': self.encryptor = aes_ctr_encryptor(self.key, self.nonce) self.encoder = base64.encodestring self.encode_batches = self.FILTER_BLOCKSIZE elif self.cipher == 'none': self.encryptor = lambda d: d self.encoder = base64.encodestring self.encode_batches = self.FILTER_BLOCKSIZE elif self.cipher == 'broken': self.encoder = self.encryptor = lambda d: d self.encode_batches = None else: self.encoder = self.encryptor = None self.encode_batches = None ChecksummingStreamer.__init__(self, dir=dir, name=name, long_running=long_running, use_filter=use_filter) # These are necessary for two reasons: # # 1. Prevent the public checksum from revealing what was encrypted. # 2. When using malleable encryption modes (such as CTR), make it # infeasable for attackers to generate a checksum that validates # modified data. # # As this is for data at rest, we are not particularly concerned with # oracle attacks. If an attacker has access to your mailpile on disk # while it is in use (so they can inject new messages), there are # other attacks which are both easier and more severe. # self.inner_sha256 = None self.inner_sha = hashlib.sha256() self.inner_sha.update(self.key) self.inner_sha.update(self.nonce or '') self._send_key() def _write_filter(self, data): if data: self.inner_sha.update(data) if self.encryptor and (data or self.encode_buffer): if self.encode_batches: eof = not data if data: self.encode_buffer += data data = '' for i in (512, 128, 8, 1): batch = i * self.encode_batches while eof or (len(self.encode_buffer) >= batch): d = self.encode_buffer[:batch] b = self.encode_buffer[batch:] self.encode_buffer = b data += self.encoder(self.encryptor(d)) eof = False return data def _sha256_callback(self, data): return ChecksummingStreamer._sha256_callback(self, data) def outer_mac_sha256(self): return mac_sha256(self.key or '', self.outer_sha.digest()) def write_pad_and_flush(self, data, pad=' '): if self.encryptor and (data or self.encode_buffer): if self.encode_batches: remainder = len(self.encode_buffer) + len(data) remainder %= self.encode_batches padding = self.encode_batches - remainder data += (pad * padding) self.write(data) self.flush() def finish(self, *args, **kwargs): if not self.finished: while self.encode_buffer: self.write('') rv = ChecksummingStreamer.finish(self, *args, **kwargs) self._write_inner_sha256() return rv else: return ChecksummingStreamer.finish(self, *args, **kwargs) def _write_inner_sha256(self): if not self.inner_sha256: self.inner_sha256 = mac_sha256(self.key, self.inner_sha.digest()) pos = self.tempfile.tell() self.tempfile.seek(0, 0) old_data = self.tempfile.read(4096) sha_256_header = SHA_256_FORMAT % (self.inner_sha256, ) new_data = re.sub(SHA_256_RE, sha_256_header, old_data) if old_data != new_data: self.tempfile.seek(0, 0) self.tempfile.write(new_data) self.tempfile.seek(pos, 0) def _nonce_and_mutated_key(self, key): # This generates a nonce which may be used as a salt, IV, or # counter-prefix depending the algorithm and mode in use. We # also use it to derive a mutated key for each message, thus # reducing the risks of the (key, iv) pairs ever repeating even # if a mistake is made somewhere else. nonce = '%32.32x' % getrandbits(32 * 4) return nonce, genkey(key, nonce)[:32].strip() def _send_key(self): # We talk directly to the underlying FD, to avoid corrupting the # inner MD5 sum (calculated using the _write_filter() above). if not self.encryptor: self._fd.write('%s\n' % self.key) def _mk_command(self): if self.encryptor: return None return [OPENSSL_COMMAND(), "enc", "-e", "-a", "-%s" % self.cipher, "-pass", "stdin", "-bufsize", "0", "-md", OPENSSL_MD_ALG] def _write_preamble(self): self.fd.write(self.BEGIN_DATA) self.fd.write('cipher: %s\n' % self.cipher) if self.nonce: self.fd.write('nonce: %s\n' % self.nonce) self.fd.write(SHA_256_PLACEHOLDER + '\n') if self.EXTRA_HEADERS: self.fd.write(self.EXTRA_HEADERS % self.header_data) self.fd.write('\n') self.fd.flush() def _write_postamble(self): if self.END_DATA: self.fd.write('\n') self.fd.write(self.END_DATA) self.fd.flush() class EncryptingUndelimitedStreamer(EncryptingDelimitedStreamer): """ This class creates a coprocess for encrypting data. The data will be streamed to a named temporary file on disk, which can then be read back or linked to a final location. """ BEGIN_DATA = "X-Mailpile-Encrypted-Data: v2\n" EXTRA_HEADERS = ("From: Mailpile \n" "Subject: %(subject)s\n") EXTRA_DATA = {'subject': 'Mailpile encrypted data'} END_DATA = "" def EncryptingStreamer(*args, **kwargs): delimited = kwargs.get('delimited', False) if 'delimited' in kwargs: del kwargs['delimited'] if delimited: return EncryptingDelimitedStreamer(*args, **kwargs) else: return EncryptingUndelimitedStreamer(*args, **kwargs) class DecryptingStreamer(InputCoprocess): """ This class creates a coprocess for decrypting data. """ BEGIN_PGP = "-----BEGIN PGP MESSAGE-----" END_PGP = "-----END PGP MESSAGE-----" BEGIN_MED = "-----BEGIN MAILPILE ENCRYPTED DATA-----" BEGIN_MED2 = "X-Mailpile-Encrypted-Data: " END_MED = "-----END MAILPILE ENCRYPTED DATA-----" PREFERRED_CIPHER = None STATE_BEGIN = 0 STATE_HEADER = 1 STATE_DATA = 2 STATE_ONLY_DATA = 3 STATE_END = 4 STATE_RAW_DATA = 5 STATE_PGP_DATA = 6 STATE_ERROR = -1 @classmethod def StartEncrypted(cls, line): return (line.startswith(cls.BEGIN_MED) or line.startswith(cls.BEGIN_MED2) or line.startswith(cls.BEGIN_PGP)) @classmethod def EndEncrypted(cls, line): return (line.startswith(cls.END_MED) or line.startswith(cls.END_PGP)) def __init__(self, fd, mep_key=None, gpg_pass=None, sha256=None, cipher=None, name=None, long_running=False, gpgi=None, md_alg=None): self.expected_outer_sha256 = sha256 self.expected_inner_sha256 = None self.expected_inner_md5sum = None self.name = name self.outer_sha = hashlib.sha256() self.inner_sha = hashlib.sha256() self.inner_md5 = hashlib.md5() self.cipher = self.PREFERRED_CIPHER or PREFERRED_CIPHER self.md_alg = md_alg or OPENSSL_MD_ALG self.state = self.STATE_BEGIN self.buffered = '' self.mep_version = None self.mep_mutated = None self.mep_key = mep_key self.gpg_pass = gpg_pass self.gpgi = gpgi self.decryptor = None self.decoder = None self.decoder_data_bytes = 0 # Not counting white-space # Start reading our data... self.startup_lock = CryptoLock() self.startup_lock.acquire() self.data_filter = self._mk_data_filter(fd, self._read_data, self.startup_lock.release) self.read_fd = self.data_filter.reader() try: # Once the header has been processed (_read_data() will release # the lock), fork out our coprocess. self.startup_lock.acquire() InputCoprocess.__init__(self, self._mk_command(), self.read_fd, name=name, long_running=long_running) except: try: self.data_filter.join(aborting=True) self.data_filter.close() except (IOError, OSError): pass raise finally: self.startup_lock.release() self.startup_lock = None def _read_filter(self, data): if data: if self.expected_inner_sha256: self.inner_sha.update(data) if self.expected_inner_md5sum: self.inner_md5.update(data) return data def close(self): self.read_fd.close() self.data_filter.join() return InputCoprocess.close(self) def verify(self, testing=False, _raise=None): if self.close() != 0: if testing: print('Close returned nonzero') if _raise: raise _raise('Non-zero exit code from coprocess') return False if self.expected_inner_sha256: mac = mac_sha256(self.mep_mutated, self.inner_sha.digest()) if self.expected_inner_sha256 != mac: if testing: print('Inner %s != %s' % (self.expected_inner_sha256, mac)) if _raise: raise _raise('Invalid inner SHA256') return False elif self.expected_inner_md5sum: if self.expected_inner_md5sum != self.inner_md5.hexdigest(): if testing: print('Inner %s != %s' % (self.expected_inner_md5sum, self.inner_md5.hexdigest())) if _raise: raise _raise('Invalid inner MD5 sum') return False elif testing and not self.expected_inner_md5sum: print('No inner MD5 sum or SHA256 expected') if self.expected_outer_sha256: mac = mac_sha256(self.mep_mutated, self.outer_sha.digest()) if self.expected_outer_sha256 != mac: if testing: print('Outer %s != %s' % (self.expected_outer_sha256, mac)) if _raise: raise _raise('Invalid outer SHA256') return False elif testing and not self.expected_outer_sha256: print('No outer SHA256 expected') return True def _mk_data_filter(self, fd, cb, ecb): return IOFilter(fd, cb, error_callback=ecb, name='%s/filter' % (self.name or 'ds')) def _read_data(self, data): def process(data): if self.decryptor is not None: eof = not data if self.decoder_data_bytes and data: self.buffered += ''.join([c for c in data if c not in (' ', '\t', '\r', '\n')]) else: self.buffered += (data or '') data = '' for i in (256, 64, 8, 1): batch = max(1, i * self.decoder_data_bytes) while eof or (len(self.buffered) >= batch): if self.decoder_data_bytes: d = self.buffered[:batch] b = self.buffered[batch:] self.buffered = b else: d, self.buffered = self.buffered, '' try: data += self.decryptor(self.decoder(d)) eof = False except TypeError: raise IOError('%s: Bad data, failed to decode' % self.name) return (data or '') if data is None: # EOF! if self.state in (self.STATE_BEGIN, self.STATE_HEADER): self.state = self.STATE_RAW_DATA self.startup_lock.release() data, self.buffered = self.buffered, '' return process(data) + process(None) return process(None) if self.expected_outer_sha256: # The outer MD5 sum is calculated over all data, but with any # CRLF sequences normalized to only LF and the sha256 header # itself replaced with a placeholder. if self.state in (self.STATE_BEGIN, self.STATE_HEADER): sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER, re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data)) else: sum_data = data sum_data = sum_data.replace('\r', '').replace('\n', '\r\n') self.outer_sha.update(sum_data) if self.state in ( self.STATE_RAW_DATA, self.STATE_PGP_DATA, self.STATE_ONLY_DATA): return process(data) if self.state == self.STATE_BEGIN: self.buffered += data if (len(self.buffered) >= len(self.BEGIN_PGP) and self.buffered.startswith(self.BEGIN_PGP)): self.state = self.STATE_PGP_DATA if self.gpg_pass: self.gpg_pass.seek(0, 0) passphrase, c = [], self.gpg_pass.read(1) while c != '': passphrase.append(c) c = self.gpg_pass.read(1) self.startup_lock.release() return ''.join(passphrase + ['\n', self.buffered]) else: self.startup_lock.release() return self.buffered # Note: The max() check is OK, because both formats add more # data which covers the difference. if len(self.buffered) >= max(len(self.BEGIN_MED), len(self.BEGIN_MED2)): if not (self.buffered.startswith(self.BEGIN_MED) or self.buffered.startswith(self.BEGIN_MED2)): self.state = self.STATE_RAW_DATA self.startup_lock.release() return self.buffered if '\r\n\r\n' in self.buffered: header, data = self.buffered.split('\r\n\r\n', 1) headlines = header.strip().split('\r\n') self.state = self.STATE_HEADER elif '\n\n' in self.buffered: header, data = self.buffered.split('\n\n', 1) headlines = header.strip().split('\n') self.state = self.STATE_HEADER else: return '' else: return '' if self.state == self.STATE_HEADER: # State: header and data have been set, header is complete. self.buffered = '' headers = dict([l.split(': ', 1) for l in headlines if ': ' in l]) nonce = headers.get('nonce', '') self.mep_mutated = self._mutate_key(self.mep_key, nonce) self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1') self.cipher = headers.get('cipher', self.cipher) data_fmt = '%s:%s' % (self.mep_version, self.cipher) if data_fmt != PREFERRED_FORMAT: DETECTED_OBSOLETE_FORMATS.add(data_fmt) eim = self.expected_inner_md5sum = headers.get('md5sum') if eim and eim == ('0' * LEN_MD5_SUM): self.expected_inner_md5sum = None if self.expected_inner_md5sum: self.inner_md5.update(self.mep_mutated) self.inner_md5.update(nonce) eis = self.expected_inner_sha256 = headers.get('sha256') if eis and eis == ('0' * LEN_SHA_256): self.expected_inner_sha256 = None if self.expected_inner_sha256: self.inner_sha.update(self.mep_mutated) self.inner_sha.update(nonce) if self.buffered.startswith(self.BEGIN_MED2): self.state = self.STATE_ONLY_DATA else: self.state = self.STATE_DATA if self.cipher == 'aes-128-ctr': self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce) self.decoder = base64.b64decode # Decode data in chunks this big (multiple of 4 and 16); # guarantees workable chunks for both AES-CTR and base64. self.decoder_data_bytes = 32 * 1024 elif self.cipher == 'none': self.decryptor = lambda d: d self.decoder = base64.b64decode self.decoder_data_bytes = 32 * 1024 elif self.cipher == 'broken': self.decryptor = lambda d: d self.decoder = lambda d: d self.expected_inner_md5sum = None self.expected_inner_sha256 = None else: self.decryptor = None data = '\n'.join((self.mep_mutated, data)) self.startup_lock.release() if self.state == self.STATE_ONLY_DATA: return process(data) if self.state == self.STATE_DATA: for delim in (self.END_MED, self.END_PGP): if delim in data: for pf in ('\r\n', '\n', ''): if pf + delim in data: data = data.split(pf + delim, 1)[0] self.state = self.STATE_END return process(data) return process(data) # Error, end and unknown states... return '' def _mutate_key(self, key, nonce): return genkey(key or '', nonce)[:32].strip() def _mk_command(self): if self.state == self.STATE_RAW_DATA: return None elif self.decryptor is not None: return None elif self.state == self.STATE_PGP_DATA: safe_assert(self.gpgi is not None) if self.gpg_pass: return self.gpgi.common_args(will_send_passphrase=True) else: return self.gpgi.common_args() return [OPENSSL_COMMAND(), "enc", "-d", "-a", "-%s" % self.cipher, "-pass", "stdin", "-md", self.md_alg] class PartialDecryptingStreamer(DecryptingStreamer): def __init__(self, start_data, *args, **kwargs): self.start_data = start_data DecryptingStreamer.__init__(self, *args, **kwargs) def _mk_data_filter(self, fd, cb, ecb): return ReadLineIOFilter(fd, cb, start_data=self.start_data, stop_check=self.EndEncrypted, error_callback=ecb, name='%s/rlfilter' % (self.name or 'ds')) if __name__ == "__main__": import random # See! Not in the main module! import StringIO def _assert(val, want=True, msg='assert'): if isinstance(want, bool): if (not val) == (not want): want = val if val != want: raise AssertionError('%s(%s==%s)' % (msg, val, want)) LEGACY_TEST_KEY = 'test key' LEGACY_PLAINTEXT = 'Hello world! This is great!\nHooray, lalalalla!\n' LEGACY_TEST_1 = """\ X-Mailpile-Encrypted-Data: v1 cipher: aes-256-cbc nonce: SEefbOfc9UQmZeWWGWQMrb0n6czXY2Uv md5sum: b07d3ed58b79a69ab5496cffcab5d878 From: Mailpile Subject: Mailpile encrypted data U2FsdGVkX18zVuMErdegtGziWDLhSvNRb7YRRxmYKMmygI1H3bp+mXffToii6lGB Z7Vlo78g20D8NAO6dpJfmA== """ LEGACY_TEST_2 = """\ -----BEGIN MAILPILE ENCRYPTED DATA----- cipher: aes-256-cbc nonce: SB+fmmM72oFpf/FO4wnaHhFBvhgzpbwW md5sum: 90dfb2850da49c8a6027415521dadb3c U2FsdGVkX19U8G7SKp8QygUusdHZThlrLcI04+jZ9U5kwfsw7bJJ2721dwgIpCUh 3wpQjsYtFF2dcKBjrG7xyw== -----END MAILPILE ENCRYPTED DATA----- """ # Do this before checking for fd leaks, as it may open up /dev/urandom # and keep it open. b = getrandbits(128) # Create a pipe, this tells us which FDs are available next fdpair1 = os.pipe() for fd in fdpair1: os.close(fd) def fdcheck(where): fdpair2 = os.pipe() try: for fd in fdpair2: if fd not in fdpair1: print('Probably have an FD leak at %s!' % where) print('Verify with: lsof -g %s' % os.getpid()) import time time.sleep(900) return False return True finally: for fd in fdpair2: os.close(fd) bc = [0] def counter(data): bc[0] += len(data or '') return (data or '') # Cleanup... try: os.unlink('/tmp/iofilter.tmp') except OSError: pass print('Test the IOFilter in write mode') with open('/tmp/iofilter.tmp', 'w') as bfd: with IOFilter(bfd, counter) as iof: iof.writer().write('Hello world!') with open('/tmp/iofilter.tmp', 'r') as iof: assert(iof.read() == 'Hello world!') _assert(bc[0], 12) _assert(fdcheck('IOFilter in write mode')) print('Test the IOFilter in read mode') bc[0] = 0 with open('/tmp/iofilter.tmp', 'r') as bfd: with IOFilter(bfd, counter) as iof: data = iof.reader().read() _assert(data, 'Hello world!') _assert(bc[0], 12) _assert(fdcheck('IOFilter in read mode')) print('Test the IOFilter in incomplete read mode') bc[0] = 0 with open('/dev/urandom', 'r') as bfd: with IOFilter(bfd, counter) as iof: data = iof.reader().read(4096) _assert(bc[0] >= 4096, msg='%s >= 4096' % bc[0]) _assert(len(data) == 4096) _assert(fdcheck('IOFilter in incomplete read mode')) print('Test the ReadLineIOFilter in incomplete read mode') bc[0], daemonlogline = 0, '' with open('/etc/passwd', 'r') as bfd: with IOFilter(bfd, counter) as iof: for line in iof.reader(): if 'daemon' in line: daemonlogline = line break _assert(bc[0] > 80, msg='%s > 80' % bc[0]) _assert('daemon' in daemonlogline, msg='daemon in %s' % daemonlogline) _assert(fdcheck('ReadLineIOFilter in incomplete read mode')) print('Null decryption test, sha256 verification only') outer_mac_sha256 = '7982970534e089b839957b7e174725ce1878731ed6d700766e59cb16f1c25e27' with open('/tmp/iofilter.tmp', 'rb') as bfd: with DecryptingStreamer(bfd, mep_key='test key', sha256=outer_mac_sha256 ) as ds: _assert('Hello world!', ds.read()) _assert(ds.verify(testing=True)) _assert(fdcheck('Decrypting test, sha256 verification')) print('Legacy (MEP v1) decryption test') for legacy in (LEGACY_TEST_1, LEGACY_TEST_2): lfd = StringIO.StringIO(legacy) with PartialDecryptingStreamer([], lfd, mep_key=LEGACY_TEST_KEY) as ds: plaintext = '' d = ds.read(1) while len(d) > 0: plaintext += d d = ds.read(random.randint(10, max(11, len(legacy)))) try: _assert(plaintext, LEGACY_PLAINTEXT) _assert(ds.verify(testing=True)) except AssertionError: print('command=%s' % ds.command) print('stderr=%s' % ds.stderr) print('key=%s [%s]\n%s' % (LEGACY_TEST_KEY, ds.mep_mutated, legacy)) raise for cipher in ('none', 'broken', 'aes-128-ctr', 'aes-256-cbc'): for filter_sha256 in (True, False): for delim in (True, False): print(('Encryption test, cipher=%s, delim=%s, filter_sha256=%s' ) % (cipher, delim, filter_sha256)) fn = '/tmp/enc-%s-%s-%s.tmp' % (cipher, delim, filter_sha256) with open(fn, 'wb') as fd: fd.write('junk') # Make sure overwriting works t0 = time.time() mul = 123400 data = 'Hello world! This is great!\nHooray, lalalalla!\n' * mul parts = [(data, 'wb')] if delim: for i in (2, 5, 10, mul // 10, mul): more = 'part two, yeaaaah\n' * (mul // i) parts.append((more, 'ab')) data += more encrypted = [] for part, mode in parts: with EncryptingStreamer('test key', dir='/tmp', delimited=delim, cipher=cipher, use_filter=filter_sha256) as es: d = part while d: i = min(len(d), random.randint(10, max(11, len(d)))) es.write(d[:i]) d = d[i:] es.finish() es.save(fn, mode=mode) encrypted.append(es.outer_mac_sha256()) _assert(fdcheck('Encrypted data, delimited=%s' % delim)) t1 = time.time() print('Decryption test, delim=%s' % delim) with open(fn, 'rb') as bfd: new_data = '' for ms in encrypted: if delim: ds = PartialDecryptingStreamer( [], bfd, mep_key='test key', sha256=ms) else: ds = DecryptingStreamer( bfd, mep_key='test key', sha256=ms) with ds: d = ds.read(9999) while d: new_data += d d = ds.read(random.randint(10, 102400)) _assert(ds.verify(testing=True)) try: _assert(data, new_data) except: print('OLD %d bytes vs. NEW %d bytes: \n%s\n' % ( len(data), len(new_data), new_data[-100:])) raise _assert(fdcheck('Decrypting test, delimited=%s' % delim)) t2 = time.time() print (' => Elapsed: %.3fs + %.3fs = %.3fs (%.2f MB/s)' % (t1-t0, t2-t1, t2-t0, len(new_data)/(1024*1024*(t2-t0)))) # Cleanup os.unlink(fn) print() _assert(len(DETECTED_OBSOLETE_FORMATS) > 0) print('Obsolete formats detected: %s' % DETECTED_OBSOLETE_FORMATS) os.unlink('/tmp/iofilter.tmp') _assert(fdcheck('All done')) ================================================ FILE: mailpile/crypto/tor.py ================================================ import copy import socket import subprocess import threading import time import traceback import re import sys import os import stem.process import stem.control import mailpile.util from mailpile.eventlog import Event from mailpile.platforms import RandomListeningPort, GetDefaultTorPath from mailpile.safe_popen import PresetSafePopenArgs, MakePopenSafe from mailpile.util import okay_random if 'pythonw' in sys.executable: debug_target = open( os.devnull, 'w' ) else: debug_target = sys.stdout def debug( text ): debug_target.write( text + '\n' ) debug_target.flush() # Version check for STEM >= 1.4 assert(int(stem.__version__[0]) > 1 or (int(stem.__version__[0]) == 1 and int(stem.__version__[2]) >= 4)) class Tor(threading.Thread): _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(Tor, cls).__new__(cls, *args, **kwargs) return cls._instance def __init__(self, session=None, config=None, socks_port=None, control_port=None, tor_binary=None, callbacks=None): threading.Thread.__init__(self) self.session = session self.config = config or (session.config if session else None) self.callbacks = callbacks if self.config is None: self.socks_port = None self.control_port = None self.control_password = okay_random(32) self.tor_binary = tor_binary else: self.socks_port = self.config.sys.tor.socks_port self.control_port = self.config.sys.tor.ctrl_port self.control_password = self.config.sys.tor.ctrl_auth self.tor_binary = tor_binary or self.config.sys.tor.binary or None if socks_port is not None: self.socks_port = socks_port if control_port is not None: self.control_port = control_port self.event = Event(source=self, flags=Event.INCOMPLETE, data={}) self.lock = threading.Lock() self.tor_process = None self.tor_controller = None self.hidden_services = {} self.keep_looping = True self.started = False def run(self): starts = 0 while self.keep_looping and not mailpile.util.QUITTING: starts += 1 try: self._run_once() except OSError: pass for i in range(0, 5 * min(60, starts)): if mailpile.util.QUITTING: break time.sleep(0.2) def _run_once(self): try: random_ports = RandomListeningPort(count=2) with self.lock: self.event.flags = Event.INCOMPLETE self.tor_process = 'starting up' self.tor_controller = None self.started = True if not self.socks_port: self.socks_port = random_ports[0] if not self.control_port: self.control_port = random_ports[1] if self.tor_binary is None: self.tor_binary = GetDefaultTorPath() tor_process_config = { 'SocksPort': str(self.socks_port), 'ControlPort': str(self.control_port), 'HashedControlPassword': self._hashed_control_password()} self._log_line('Launching Tor (%s)' % self.tor_binary, notify=True) PresetSafePopenArgs(long_running=True) self.tor_process = stem.process.launch_tor_with_config( timeout=None, # Required or signal.signal will raise tor_cmd=self.tor_binary, config= tor_process_config, init_msg_handler=self._log_line) ctrl = stem.control.Controller.from_port(port=self.control_port) ctrl.authenticate(password=self.control_password) self.tor_controller = ctrl self.event.flags = Event.RUNNING self._log_line('Tor is live on socks=%d, control=%d' % (self.socks_port, self.control_port), notify=True) finally: MakePopenSafe() # Relaunch all the hidden services, if our Tor process died on us. self.relaunch_hidden_services() # Invoke any on-startup callbacks for cb in (self.callbacks or []): try: cb(self) except: self._log_line('Callback %s failed: %s' % (cb, traceback.format_exc()), notify=True) # Finally, just wait until our child process terminates. try: self.tor_process.wait() except: pass finally: self.event.flags = Event.COMPLETE self._log_line('Shut down', notify=True) self.tor_controller = None self.tor_process = None def _hashed_control_password(self): try: hasher = subprocess.Popen( [self.tor_binary, '--hush', '--hash-password', str(self.control_password)], stdout=subprocess.PIPE, bufsize=1) hasher.wait() expr = re.compile('([\d]{2}:[\w]{58})') match = filter(None, map(expr.match, hasher.stdout))[0] passhash = match.group(1) return passhash except: return None def _log_line(self, line, notify=False): if self.session: if notify: self.session.ui.notify(line) else: self.session.ui.debug(line) else: debug('%s' % line) log = self.event.data.get('log', []) log.append(line.strip()) if len(log) > 100: log[:10] = [] self.event.data['log'] = log if notify: self.event.message = line if self.config and self.config.event_log: self.config.event_log.log_event(self.event) def relaunch_hidden_services(self): hidden_services = copy.copy(self.hidden_services) for onion, (portmap, key_t, key_c) in hidden_services.iteritems(): if key_t and key_c: self.launch_hidden_service(portmap, key_t, key_c) else: self._log_line('Failed to relaunch: %s' % onion, notify=True) def launch_hidden_service(self, portmap, key_type=None, key_content=None): with self.lock: aor = self.tor_controller.create_ephemeral_hidden_service( portmap, key_type=key_type if (key_content and key_type) else 'NEW', key_content=key_content or 'BEST', detached=True, await_publication=True) self.hidden_services[aor.service_id] = ( portmap, aor.private_key_type or key_type, aor.private_key or key_content) self._log_line('Listening on Onion: %s.onion' % aor.service_id, notify=True) return aor def stop_tor(self, wait=True): self.keep_looping = False if self.tor_process is not None: try: for onion in self.hidden_services: try: t.tor_controller.remove_ephemeral_hidden_service(onion) except: pass with self.lock: self.tor_process.kill() if wait: self.tor_process.wait() except: pass Tor._instance = None def isReady(self, wait=False): while True: if ((self.tor_process is not None) and (self.tor_controller is not None)): return True if not wait: return False with self.lock: # If we have the lock, but self.tor_process is None, that # means startup has failed or we are dead - stop waiting! wait = (not self.started) or (self.tor_process is not None) if not self.started: time.sleep(0.1) def isAlive(self): # FIXME: This is inaccurate return (self.tor_process is not None) def quit(self, join=True): self.stop_tor(wait=join) if __name__ == "__main__": TEST_KEY = "RSA1024:MIICXQIBAAKBgQDQ/+aYFpSvZZ5Ce2cpsuJz1epCcY9n+HZx/bC/D7mqEXCdDB9W13FMuwwK9FvjjXdfJzkdJ1GEcppEzd69C5xPZo2k+klKDhMONYhGHcm+CGu+JWNbqrcInNfZageu1Hg8g5Kz2h+/xCmuqKLSxGwJGvIoYfZupyn3DaxGnZv/2QIDAQABAoGAYs13L9MM+1Yo2PkJrhbZIzWvhzW0O8ykAgOSeOBwP0v7VuMSNbWn5ERQzyTyA8Mu+ZbLU1LxIJIlB/3jHK/Odoe2kkPjjaeKKVXGM+NMefps/YPs8abql06YoWN6KshY0BYkzkmlF/Xxl4t+jjvDG9Fsx6kJV6LKRwm6BFVzUTkCQQD2ujGQs1I1fsuCCHZXcnyoLO/hJaJLoj3clCiYcWhnfdpgkv0+CdIYE+DPZNsiIfruQQYZzZjKtO2xx0fvqKuPAkEA2Nq46nR1L0ISJizSfTYkz+KXuV8kgkMxwzkZC6l+DAqf4qxjFcDOwrIw9f75N8DcveHLD4R/fyMaesW6SK8KFwJAZ4Om1/bkPt17tIqoW/gEpOp1mhiYBvOC0NC4V3z9OK5suKfy59xm8QMmBt1hsuhexycw0BKaUDGoqDXb0IkLsQJBANQaUsWXdMrtX90Q+CxaGfVvVyGL6qSyXmjpXxLmDBBxD+Ng42VyeYk7SuJBKreanw3mXHvoB+BtkEfHQCY5dq8CQQCofoToQr5mTrlomus6/ei22Ein/BS9s0YUPCOpMkZfSp/GaWyEH7QjxatM/LoaMRlH/Y/wGMEK8P05F9DGBtSP" t = Tor() try: debug_target = open( sys.argv[1], 'w' ) except IndexError: pass try: debug( "*** Starting Tor" ) t.start() if t.isReady(wait=True): debug( "*** Creating hidden services..." ) key_type, key_content = TEST_KEY.split(':', 1) aor = t.launch_hidden_service({80: 80}, key_type, key_content) aor = t.launch_hidden_service(443) #raw_input("*** Hit enter to disable service and shutdown ***") debug("waiting...") time.sleep(5) finally: debug("quiting!?!") t.quit() debug("exiting!?!") ================================================ FILE: mailpile/eventlog.py ================================================ from __future__ import print_function import copy import datetime import json import os import threading import time from email.utils import formatdate, parsedate_tz, mktime_tz from mailpile.crypto.streamer import EncryptingStreamer, DecryptingStreamer from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.util import EventRLock, EventLock, CleanText, json_helper from mailpile.util import safe_remove, thread_context EVENT_COUNTER_LOCK = threading.Lock() EVENT_COUNTER = 0 def NewEventId(): """ This is guaranteed to generate unique event IDs for up to 1 million events per second. Beyond that, all bets are off. :-P """ global EVENT_COUNTER with EVENT_COUNTER_LOCK: EVENT_COUNTER = EVENT_COUNTER+1 EVENT_COUNTER %= 0x100000 return '%8.8x-%5.5x-%x' % (time.time(), EVENT_COUNTER, os.getpid()) def _ClassName(obj, ignore_regexps=False): if isinstance(obj, (str, unicode)): return str(obj).replace('mailpile.', '.') elif hasattr(obj, '__classname__'): return str(obj.__classname__).replace('mailpile.', '.') elif ignore_regexps and 'SRE_Pattern' in str(obj.__class__): return obj else: module = str(obj.__class__.__module__) if module.startswith('mailpile.'): module = module[len('mailpile'):] return '%s.%s' % (module, str(obj.__class__.__name__)) class Event(object): """ This is a single event in the event log. Actual interpretation and rendering of events should be handled by the respective source class. """ RUNNING = 'R' COMPLETE = 'c' INCOMPLETE = 'i' FUTURE = 'F' # For now these live here, we may templatize this later. PREAMBLE_HTML = '
    ' PUBLIC_HTML = ('
  • %(date)s ' '%(message)s
    • ') PRIVATE_HTML = PUBLIC_HTML POSTAMBLE_HTML = '
' @classmethod def Parse(cls, json_string): try: return cls(*json.loads(json_string)) except: return cls() def __init__(self, ts=None, event_id=None, flags='c', message='', source=None, data=None, private_data=None): self._data = [ '', (event_id or NewEventId()).replace('.', '-'), flags, message, _ClassName(source), data or {}, private_data or {}, ] self._set_ts(ts or time.time()) def __str__(self): return json.dumps(self._data, default=json_helper) def _set_ts(self, ts): if hasattr(ts, 'timetuple'): self._ts = int(time.mktime(ts.timetuple())) elif isinstance(ts, (str, unicode)): self._ts = int(mktime_tz(parsedate_tz(ts))) else: self._ts = float(ts) self._data[0] = formatdate(self._ts) def _set(self, col, value): self._set_ts(time.time()) self._data[col] = value def _get_source_class(self): try: module_name, class_name = CleanText(self.source, banned=CleanText.NONDNS ).clean.rsplit('.', 1) if module_name.startswith('.'): module_name = 'mailpile' + module_name module = __import__(module_name, globals(), locals(), class_name) return getattr(module, class_name) except (ValueError, AttributeError, ImportError): return None date = property(lambda s: s._data[0], lambda s, v: s._set_ts(v)) ts = property(lambda s: s._ts, lambda s, v: s._set_ts(v)) event_id = property(lambda s: s._data[1], lambda s, v: s._set(1, v)) flags = property(lambda s: s._data[2], lambda s, v: s._set(2, v)) message = property(lambda s: s._data[3], lambda s, v: s._set(3, v)) source = property(lambda s: s._data[4], lambda s, v: s._set(4, _ClassName(v))) data = property(lambda s: s._data[5], lambda s, v: s._set(5, v)) private_data = property(lambda s: s._data[6], lambda s, v: s._set(6, v)) source_class = property(_get_source_class) def as_dict(self, private=True): try: return self.source_class.EventAsDict(self, private=private) except (AttributeError, NameError): data = { 'ts': self.ts, 'date': self.date, 'event_id': self.event_id, 'message': self.message, 'flags': self.flags, 'source': self.source, 'data': self.data } if private: data['private_data'] = self.private_data return data def as_text(self, private=True, compact=False): try: return self.source_class.EventAsText(self, private=private, compact=True) except (AttributeError, NameError): if compact: return '%s=%s:%s %s' % (self.event_id, self.source.split('.')[-1], self.flags, self.message) else: return json.dumps(self.as_dict(private=private), default=json_helper) def as_json(self, private=True): try: return self.source_class.EventAsJson(self, private=private) except (AttributeError, NameError): return json.dumps(self.as_dict(private=private)) def as_html(self, private=True): try: return self.source_class.EventAsHtml(self, private=private) except (AttributeError, NameError): if private: return self.PRIVATE_HTML % self.as_dict(private=True) else: return self.PUBLIC_HTML % self.as_dict(private=False) def GetThreadEvent(create=False, message=None, source=None): ctx = thread_context() if ctx and 'event' in ctx[-1]: return ctx[-1]['event'] elif create: return Event(message=message, source=source) else: return None class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ KEEP_LOGS = 2 def __init__(self, logdir, decryption_key_func, encryption_key_func, rollover=1024): self.logdir = logdir self.decryption_key_func = decryption_key_func or (lambda: None) self.encryption_key_func = encryption_key_func or (lambda: None) self.rollover = rollover self._events = {} # Internals... self._watching_uis = [] self._waiter = threading.Condition(EventRLock()) self._lock = EventLock() self._log_fd = None def _notify_waiters(self): with self._waiter: self._waiter.notifyAll() def wait(self, timeout=None): with self._waiter: self._waiter.wait(timeout) def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir, name='EventLog/ES', use_filter=False, long_running=True) self._log_fd.save(self._save_filename(), finish=False) self._log_write = self._log_fd.write_pad_and_flush else: self._log_fd = open(self._save_filename(), 'wb', 0) self._log_write = self._log_fd.write # Write any incomplete events to the new file for e in self.incomplete(): self._log_write('%s\n') # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self._events = kept_events self._open_log() self.purge_old_logfiles() def _list_logfiles(self): return sorted([l for l in os.listdir(self.logdir) if not l.startswith('.')]) def _save_events(self, events, recursed=False): if not self._log_fd: self._open_log() events.sort(key=lambda ev: ev.ts) try: for event in events: self._log_write('%s\n' % event) self._events[event.event_id] = event except IOError: if recursed: raise else: self._unlocked_close() return self._save_events(events, recursed=True) def _load_logfile(self, lfn): enc_key = self.decryption_key_func() with open(os.path.join(self.logdir, lfn)) as fd: if enc_key: with DecryptingStreamer(fd, mep_key=enc_key, name='EventLog/DS(%s)' % lfn ) as streamer: lines = streamer.read() streamer.verify(_raise=IOError) else: lines = fd.read() if lines: for line in lines.splitlines(): event = Event.Parse(line.strip()) self._events[event.event_id] = event def _match(self, event, filters): def compare(val, rule): if isinstance(rule, (str, unicode)): return unicode(val) == unicode(rule) else: return rule.match(unicode(val)) is not None for kw, rule in filters.iteritems(): if kw.endswith('!'): truth, okw, kw = False, kw, kw[:-1] else: truth, okw = True, kw if kw == 'source': if truth != compare(event.source, _ClassName(rule, ignore_regexps=True)): return False elif kw == 'flag': if truth != (rule in event.flags): return False elif kw == 'flags': if truth != compare(event.flags, rule): return False elif kw == 'event_id': if truth != compare(event.event_id, rule): return False elif kw == 'since': when = float(rule) if when < 0: when += time.time() if truth != (event.ts > when): return False elif kw.startswith('data_'): if truth != compare(event.data.get(kw[5:]), rule): return False elif kw.startswith('private_data_'): if truth != compare(event.data.get(kw[13:]), rule): return False else: # Unknown keywords match nothing... print('Unknown keyword: `%s=%s`' % (okw, rule)) return False return True def incomplete(self, **filters): """Return all the incomplete events, in order.""" if 'event_id' in filters: ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and Event.COMPLETE not in e.flags and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" if ts < 0: ts += time.time() if 'event_id' in filters and filters['event_id'][:1] != '!': ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def events(self, **filters): return self.since(0, **filters) def get(self, event_id, default=None): return self._events.get(event_id, default) def log_event(self, event): """Log an Event object.""" with self._lock: self._save_events([event]) self._logged += 1 self._maybe_rotate_log() self._notify_waiters() for ui in self._watching_uis: ui.notify(event.as_text(compact=True)) return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): with self._lock: return self._unlocked_close() def _unlocked_close(self): try: self._log_fd.close() self._log_fd = None except (OSError, IOError): pass def _prune_completed(self): for event_id in self._events.keys(): if Event.COMPLETE in self._events[event_id].flags: del self._events[event_id] def ui_watch(self, ui): while ui.log_parent is not None: ui = ui.log_parent if ui not in self._watching_uis: self._watching_uis.append(ui) return True else: return False def ui_unwatch(self, ui): while ui.log_parent is not None: ui = ui.log_parent try: self._watching_uis.remove(ui) except ValueError: pass def load(self): with self._lock: self._open_log() for lf in self._list_logfiles()[-4:]: try: self._load_logfile(lf) except (OSError, IOError): # Nothing we can do, no point complaining... pass self._prune_completed() self._save_events(self._events.values()) return self def purge_old_logfiles(self, keep=None): keep = keep or self.KEEP_LOGS for lf in self._list_logfiles()[:-keep]: try: safe_remove(os.path.join(self.logdir, lf)) except OSError: pass ================================================ FILE: mailpile/httpd.py ================================================ # # Mailpile's built-in HTTPD # ############################################################################### import Cookie import cStringIO import hashlib import gzip import mimetypes import os import random import select import socket import SocketServer import time import threading import traceback from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler from urllib import quote, unquote from urlparse import parse_qs, urlparse import mailpile.util import mailpile.security as security from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.urlmap import UrlMap from mailpile.util import * from mailpile.ui import * global WORD_REGEXP, STOPLIST, BORING_HEADERS, DEFAULT_PORT DEFAULT_PORT = 33411 BLOCK_HTTPD_LOCK = UiRLock() LIVE_HTTP_REQUESTS = 0 def Idle_HTTPD(allowed=1): with BLOCK_HTTPD_LOCK: sleep = 100 while (sleep and not mailpile.ui.QUITTING and LIVE_HTTP_REQUESTS > allowed): time.sleep(0.05) sleep -= 1 return BLOCK_HTTPD_LOCK class HttpRequestHandler(SimpleXMLRPCRequestHandler): # Allow persistent HTTP/1.1 connections protocol_version = 'HTTP/1.1' # We always recognize these extensions, no matter what the Python # mimetype module thinks. _MIMETYPE_MAP = dict([(ext, 'text/plain') for ext in ( 'c', 'cfg', 'conf', 'cpp', 'csv', 'h', 'hpp', 'log', 'md', 'me', 'py', 'rb', 'rc', 'txt' )] + [(ext, 'application/x-font') for ext in ( 'pfa', 'pfb', 'gsf', 'pcf' )] + [ ('css', 'text/css'), ('eot', 'application/vnd.ms-fontobject'), ('gif', 'image/gif'), ('html', 'text/html'), ('htm', 'text/html'), ('ico', 'image/x-icon'), ('jpg', 'image/jpeg'), ('jpeg', 'image/jpeg'), ('js', 'text/javascript'), ('json', 'application/json'), ('otf', 'font/otf'), ('png', 'image/png'), ('rss', 'application/rss+xml'), ('tif', 'image/tiff'), ('tiff', 'image/tiff'), ('ttf', 'font/ttf'), ('svg', 'image/svg+xml'), ('svgz', 'image/svg+xml'), ('woff', 'application/font-woff'), ]) _ERROR_CONTEXT = {'lastq': '', 'csrf': '', 'path': ''}, _NEWLINE_RE = re.compile('[\r\n]+') _HTML_RE = re.compile('[<>\'\"]+') def assert_no_newline(self, data): if re.search(self._NEWLINE_RE, str(data) or '') is not None: raise ValueError() def assert_no_html(self, data): if re.search(self._HTML_RE, data or '') is not None: raise ValueError() def send_header(self, hdr, value): self.assert_no_newline(value) return SimpleXMLRPCRequestHandler.send_header(self, hdr, value) def http_host(self): """Return the current server host, e.g. 'localhost'""" try: # rsplit removes port return self.headers.get('host', 'localhost').rsplit(':', 1)[0] except AttributeError: return 'unknown' def _load_cookies(self): """Robustified cookie parser that silently drops invalid cookies.""" cookies = Cookie.SimpleCookie() for fragment in self.headers.get('cookie', '').split('; '): if fragment: try: cookies.load(fragment) except Cookie.CookieError: pass return cookies def http_session(self): """Fetch the session ID from a cookie, or assign a new one""" session_id = self._load_cookies().get(self.server.session_cookie) if session_id: session_id = session_id.value self.assert_no_newline(session_id) else: session_id = self.server.make_session_id(self) return session_id def server_url(self): """Return the current server URL, e.g. 'http://localhost:33411/'""" try: surl = '%s://%s' % (self.headers.get('x-forwarded-proto', 'http'), self.headers.get('host', 'localhost')) self.server.server_url = surl except AttributeError: surl = self.server.server_url return surl def send_http_response(self, code, msg): """Send the HTTP response header""" msg = '%s %s' % (code, msg) self.assert_no_newline(msg) self.wfile.write('HTTP/1.1 %s\r\n' % msg) def send_http_redirect(self, destination): # We don't re-encode things here, we expect our input to already # be well formed. However, this is the last chance to block any # exploits, so we do check to make sure. self.assert_no_newline(destination) self.assert_no_html(destination) self.send_http_response(302, 'Found') body = ('

Please look here!

\n' ) % (destination,) self.wfile.write(('Location: %s\r\n' 'Content-Length: %d\r\n\r\n' ) % (destination, len(body))) self.wfile.write(body) def send_standard_headers(self, header_list=[], cachectrl='private', mimetype='text/html', x_dns_prefetch='off'): """ Send common HTTP headers plus a list of custom headers: - Cache-Control - Content-Type - X-DNS-Prefetch-Control This function does not send the HTTP/1.1 header, so ensure self.send_http_response() was called before Keyword arguments: header_list -- A list of custom headers to send, containing key-value tuples cachectrl -- The value of the 'Cache-Control' header field mimetype -- The MIME type to send as 'Content-Type' value """ if mimetype.startswith('text/') and ';' not in mimetype: mimetype += ('; charset = utf-8') self.send_header('Cache-Control', cachectrl) self.send_header('Content-Security-Policy', security.http_content_security_policy(self.server)) self.send_header('Content-Type', mimetype) self.send_header('X-DNS-Prefetch-Control', x_dns_prefetch) self.send_header('X-UA-Compatible', 'IE=Edge') # For old Windowses for header in header_list: self.send_header(header[0], header[1]) session_id = self.session.ui.html_variables.get('http_session') if session_id: cookies = Cookie.SimpleCookie() cookies[self.server.session_cookie] = session_id cookies[self.server.session_cookie]['path'] = '/' cookies[self.server.session_cookie]['max-age'] = 24 * 3600 self.send_header(*cookies.output().split(': ', 1)) if mailpile.util.QUITTING: self.send_header('Connection', 'close') self.end_headers() def send_full_response(self, message, code=200, msg='OK', mimetype='text/html', header_list=[], cachectrl=None, suppress_body=False): """ Sends the HTTP header and a response list message -- The body of the response to send header_list -- A list of custom headers to send, containing key-value tuples code -- The HTTP response code to send mimetype -- The MIME type to send as 'Content-Type' value suppress_body -- Set this to True to ignore the message parameter and not send any response body """ message = unicode(message).encode('utf-8') self.log_request(code, message and len(message) or '-') # Send HTTP/1.1 header self.send_http_response(code, msg) # Send all headers if code == 401: self.send_header('WWW-Authenticate', 'Basic realm = MP%d' % (time.time() / 3600)) # If suppress_body == True, we don't know the content length headers = [] if not suppress_body: message, headers = self._maybe_gzip(message, len(message or ''), []) self.send_standard_headers(header_list=(header_list + headers), mimetype=mimetype, cachectrl=(cachectrl or "no-cache")) # Response body if not suppress_body: self.wfile.write(message or '') def guess_mimetype(self, fpath): ext = os.path.basename(fpath).rsplit('.')[-1] return (self._MIMETYPE_MAP.get(ext.lower()) or mimetypes.guess_type(fpath, strict=False)[0] or 'application/octet-stream') def _mk_etag(self, *args): # This ETag varies by whatever args we give it (e.g. size, mtime, # etc), but is unique per Mailpile instance and should leak nothing # about the actual server configuration. data = '%s-%s' % (self.server.secret, '-'.join((str(a) for a in args))) return hashlib.md5(data).hexdigest() def _maybe_gzip(self, data, msg_size, headers): if (data and (len(data) > 1400) and (data[:2] not in ('\xff\xd8', '\x89\x50', # JPEG, PNG '\x1f\x8b', 'BZ', 'PK' # GZIP, BZIP, PKZIP )) and ('gzip' in self.headers.get('accept-encoding', ''))): gzipped = cStringIO.StringIO() with gzip.GzipFile(fileobj=gzipped, mode='w') as fd: fd.write(data) gzipped = gzipped.getvalue() if len(data) > len(gzipped): headers.extend([('Content-Length', '%s' % len(gzipped)), ('X-Full-Size', '%s' % msg_size), ('Content-Encoding', 'gzip')]) return gzipped, headers headers.append(('Content-Length', '%s' % msg_size)) return data, headers def send_file(self, config, filename, suppress_body=False): # FIXME: Do we need more security checks? if '..' in filename: code, msg = 403, "Access denied" else: try: tpl = config.sys.path.get(self.http_host(), 'html_theme') fpath, fd, mt = config.open_file(tpl, filename) with fd: mimetype = mt or self.guess_mimetype(fpath) msg_size = os.path.getsize(fpath) if not suppress_body: message = fd.read() else: message = None code, msg = 200, "OK" except IOError as e: mimetype = 'text/plain' if e.errno == 2: code, msg = 404, "File not found" elif e.errno == 13: code, msg = 403, "Access denied" else: code, msg = 500, "Internal server error" message = None msg_size = 0 # Note: We assume the actual static content almost never varies # on a given Mailpile instance, thuse the long TTL and no # ETag for conditional loads. message, headers = self._maybe_gzip(message, msg_size, []) self.log_request(code, msg_size if (message is not None) else '-') self.send_http_response(code, msg) self.send_standard_headers(header_list=headers, mimetype=mimetype, cachectrl='must-revalidate, max-age=36000') self.wfile.write(message or '') def do_POST(self, method='POST'): (scheme, netloc, path, params, query, frag) = urlparse(self.path) if path.startswith('/::XMLRPC::/'): raise ValueError(_('XMLRPC has been disabled for now.')) #return SimpleXMLRPCRequestHandler.do_POST(self) # Update thread name for debugging purposes threading.current_thread().name = 'POST:%s' % self.path.split('?')[0] self.session, config = self.server.session, self.server.session.config post_data = {} try: ue = 'application/x-www-form-urlencoded' clength = int(self.headers.get('content-length', 0)) ctype, pdict = cgi.parse_header(self.headers.get('content-type', ue)) if ctype == 'multipart/form-data': post_data = cgi.FieldStorage( fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': method, 'CONTENT_TYPE': self.headers['Content-Type']} ) elif ctype == ue: if clength > 5 * 1024 * 1024: raise ValueError(_('OMG, input too big')) post_data = cgi.parse_qs(self.rfile.read(clength), 1) else: raise ValueError(_('Unknown content-type')) except (IOError, ValueError) as e: self.send_full_response(self.server.session.ui.render_page( config, self._ERROR_CONTEXT, body='POST geborked: %s' % e, title=_('Internal Error') ), code=500) return None return self.do_GET(post_data=post_data, method=method) def do_GET(self, *args, **kwargs): global LIVE_HTTP_REQUESTS try: path = self.path.split('?')[0] threading.current_thread().name = 'WAIT:%s' % path with BLOCK_HTTPD_LOCK: LIVE_HTTP_REQUESTS += 1 threading.current_thread().name = 'WORK:%s' % path return self._real_do_GET(*args, **kwargs) finally: threading.current_thread().name = 'DONE:%s' % path LIVE_HTTP_REQUESTS -= 1 if mailpile.util.QUITTING: self.wfile.close() def _real_do_GET(self, post_data={}, suppress_body=False, method='GET'): (scheme, netloc, path, params, query, frag) = urlparse(self.path) query_data = parse_qs(query) opath = path = unquote(path) # HTTP is stateless, so we create a new session for each request. self.session, config = self.server.session, self.server.session.config server_session = self.server.session # Debugging... if 'httpdata' in config.sys.debug: self.wfile = DebugFileWrapper(sys.stderr, self.wfile) # Path manipulation... if path == '/favicon.ico': path = '%s/static/favicon.ico' % (config.sys.http_path or '') if config.sys.http_path: if not path.startswith(config.sys.http_path): self.send_full_response(_("File not found (invalid path)"), code=404, mimetype='text/plain') return None path = path[len(config.sys.http_path):] if path.startswith('/_/'): path = path[2:] for static in ('/static/', '/bower_components/'): if path.startswith(static): return self.send_file(config, path[len(static):], suppress_body=suppress_body) self.session = session = Session(config) session.ui = HttpUserInteraction(self, config, log_parent=server_session.ui) if 'context' in post_data: session.load_context(post_data['context'][0]) elif 'context' in query_data: session.load_context(query_data['context'][0]) mark_name = 'Processing HTTP API request at %s' % time.time() session.ui.start_command(mark_name, [], {}) if 'http' in config.sys.debug: session.ui.warning = server_session.ui.warning session.ui.notify = server_session.ui.notify session.ui.error = server_session.ui.error session.ui.debug = server_session.ui.debug session.ui.debug('%s: %s qs = %s post = %s' % (method, opath, query_data, post_data)) idx = session.config.index if session.config.loaded_config: name = session.config.get_profile().get('name', 'Chelsea Manning') else: name = 'Chelsea Manning' http_headers = [] http_session = self.http_session() csrf_token = security.make_csrf_token(self.server.secret, http_session) session.ui.html_variables = { 'csrf_token': csrf_token, 'csrf_field': ('' % csrf_token), 'http_host': self.headers.get('host', 'localhost'), 'http_hostname': self.http_host(), 'http_method': method, 'http_session': http_session, 'http_request': self, 'http_response_headers': http_headers, 'message_count': (idx and len(idx.INDEX) or 0), 'name': name, 'title': 'Mailpile dummy title', 'url_protocol': self.headers.get('x-forwarded-proto', 'http'), 'mailpile_size': idx and len(idx.INDEX) or 0 } session.ui.valid_csrf_token = lambda token: security.valid_csrf_token( self.server.secret, http_session, token) try: try: need_auth = not (mailpile.util.TESTING or session.config.sys.http_no_auth) commands = UrlMap(session).map( self, method, path, query_data, post_data, authenticate=need_auth) except UsageError: if (not path.endswith('/') and not session.config.sys.debug and method == 'GET'): commands = UrlMap(session).map(self, method, path + '/', query_data, post_data) url = quote(path) + '/' if query: url += '?' + query return self.send_http_redirect(url) else: raise cachectrl = None if 'http' not in config.sys.debug: etag_data = [] max_ages = [] have_ed = 0 for c in commands: max_ages.append(c.max_age()) ed = c.etag_data() have_ed += 1 if ed else 0 etag_data.extend(ed) if have_ed == len(commands): etag = self._mk_etag(*etag_data) conditional = self.headers.get('if-none-match') if conditional == etag: self.send_full_response('OK', code=304, msg='Unmodified') return None else: http_headers.append(('ETag', etag)) max_age = min(max_ages) if max_ages else 10 if max_age: cachectrl = 'must-revalidate, max-age=%d' % max_age else: cachectrl = 'must-revalidate, no-store, max-age=0' global LIVE_HTTP_REQUESTS hang_fix = 1 if ([1 for c in commands if c.IS_HANGING_ACTIVITY] ) else 0 try: LIVE_HTTP_REQUESTS -= hang_fix session.ui.mark('Running %d commands' % len(commands)) results = [cmd.run() for cmd in commands] session.ui.mark('Displaying final result') session.ui.display_result(results[-1]) finally: LIVE_HTTP_REQUESTS += hang_fix session.ui.mark('Rendering response') mimetype, content = session.ui.render_response(session.config) session.ui.mark('Sending response') self.send_full_response(content, mimetype=mimetype, header_list=http_headers, cachectrl=cachectrl) except UrlRedirectException as e: return self.send_http_redirect(e.url) except SuppressHtmlOutput: return None except AccessError: self.send_full_response(_('Access Denied'), code=403, mimetype='text/plain') return None except: e = traceback.format_exc() session.ui.debug(e) if not session.config.sys.debug: e = _('Internal Error') self.send_full_response(e, code=500, mimetype='text/plain') return None finally: session.ui.report_marks( details=('timing' in session.config.sys.debug)) session.ui.finish_command(mark_name) def do_PUT(self): return self.do_POST(method='PUT') def do_UPDATE(self): return self.do_POST(method='UPDATE') def do_HEAD(self): return self.do_GET(suppress_body=True, method='HEAD') def log_message(self, fmt, *args): if 'http' in self.server.session.config.sys.debug: self.server.session.ui.notify(self.server_url() + ' ' + (fmt % args)) class HttpServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer): def __init__(self, session, sspec, handler): SimpleXMLRPCServer.__init__(self, sspec[:2], handler) self.daemon_threads = True self.session = session self.sessions = {} self.session_cookie = None # Duplicates from SocketServer.py, so our overrides work self.__is_shut_down = threading.Event() self.__shutdown_request = False # This lets us create new HTTPDs withut waiting for this one to # completely shut down. self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # We set a large sending buffer to avoid blocking, because the GIL and # scheduling interact badly when we have busy background jobs. self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 128 * 1024) self.server_url = 'http://UNKNOWN/' self.sspec = (sspec[0] or 'localhost', self.socket.getsockname()[1], sspec[2]) # This hash includes the index ofuscation master key, which means # it should be very strongly unguessable. self.secret = okay_random(64, session.config.get_master_key()) # Generate a new unguessable session cookie name on startup while not self.session_cookie: self.session_cookie = okay_random(12, self.secret) def serve_forever(self, poll_interval=0.5, tick_func=None): """ Override SocketServer.serve_forever to allow other things to happen. """ if self.__is_shut_down is None: return self.__is_shut_down.clear() try: while not (self.__shutdown_request or mailpile.util.QUITTING): # FIXME: Let's add a global FD to interrupt this, so we can # be more responsive AND lengthen our timeouts. r, w, e = SocketServer._eintr_retry( select.select, [self], [], [], poll_interval) if self in r: self._handle_request_noblock() elif not (mailpile.util.QUITTING or tick_func is None): tick_func(self) finally: self.__shutdown_request = False if self.__is_shut_down is not None: self.__is_shut_down.set() def shutdown(self, join=True): self.__shutdown_request = True if join and (self.__is_shut_down is not None): self.__is_shut_down.wait() self.__is_shut_down = None def make_session_id(self, request): """Generate an unguessable and unauthenticated new session ID.""" session_id = None while session_id in self.sessions or session_id is None: session_id = okay_random(32, self.secret, '%s' % (request and request.headers)) return session_id def finish_request(self, request, client_address): try: SimpleXMLRPCServer.finish_request(self, request, client_address) except (socket.error, AttributeError): # AttributeError may get thrown if the underlying socket has # already been closed elsewhere and _sock = None. pass finally: if mailpile.util.QUITTING: self.shutdown() class HttpWorker(threading.Thread): def __init__(self, session, sspec): threading.Thread.__init__(self) self.httpd = HttpServer(session, sspec, HttpRequestHandler) self.daemon = True self.session = session def idle_tick(self, httpd): pass def run(self): while self.httpd is not None: try: self.httpd.serve_forever( poll_interval=1.0, tick_func=self.idle_tick) except KeyboardInterrupt: return except socket.error: pass except: time.sleep(1) if self.httpd: traceback.print_exc() def quit(self, join=False): if self.httpd: try: self.httpd.server_close() except (OSError, IOError): pass self.httpd.shutdown(join=join) self.httpd = None ================================================ FILE: mailpile/i18n.py ================================================ import gettext import os import threading from gettext import translation, gettext, NullTranslations from jinja2 import Environment, BaseLoader, TemplateNotFound ACTIVE_TRANSLATION = None RECENTLY_TRANSLATED_LOCK = threading.Lock() RECENTLY_TRANSLATED = [] FORMAT_CHECKED = {} # This little doodad will on-the-fly check whether our translators # messed up our format strings in various ways, and suppress the # translation if it is obviously broken. def _fmt_safe(translation, original): global FORMAT_CHECKED if translation in FORMAT_CHECKED: return FORMAT_CHECKED[translation] if '%' in original: try: safe_assert(len([c for c in translation if c == '%']) == len([c for c in original if c == '%'])) bogon = translation % 1 FORMAT_CHECKED[translation] = translation except TypeError: # This just means we gave the wrong argument or the wrong # number of arguments - so the format string itself is OK. FORMAT_CHECKED[translation] = translation except: FORMAT_CHECKED[translation] = original else: FORMAT_CHECKED[translation] = translation return FORMAT_CHECKED[translation] def gettext(string): with RECENTLY_TRANSLATED_LOCK: if isinstance(string, str): global RECENTLY_TRANSLATED RECENTLY_TRANSLATED = [t for t in RECENTLY_TRANSLATED[-100:] if t != string] + [string] if not ACTIVE_TRANSLATION: return string # FIXME: What if our input is utf-8? Does gettext want us to # encode it first, or send the UTF-8 string? Since we are # not encoding it, the decode below may fail. :( translation = ACTIVE_TRANSLATION.org_gettext(string) try: translation = translation.decode('utf-8') except UnicodeEncodeError: pass return _fmt_safe(translation, string) def ngettext(string1, string2, n): with RECENTLY_TRANSLATED_LOCK: global RECENTLY_TRANSLATED RECENTLY_TRANSLATED = [t for t in RECENTLY_TRANSLATED[-100:] if t not in (string1, string2) ] + [string1, string2] default = string1 if (n == 1) else string2 if not ACTIVE_TRANSLATION: return default # FIXME: What if our input is utf-8? Does gettext want us to # encode it first, or send the UTF-8 string? Since we are # not encoding it, the decode below may fail. :( translation = ACTIVE_TRANSLATION.org_ngettext(string1, string2, n) try: translation = translation.decode('utf-8') except UnicodeEncodeError: pass return _fmt_safe(translation, default) class i18n_disabler: def __init__(self): self.stack = [] def __enter__(self): global ACTIVE_TRANSLATION self.stack.append(ACTIVE_TRANSLATION) ACTIVE_TRANSLATION = None def __exit__(self, *args, **kwargs): global ACTIVE_TRANSLATION ACTIVE_TRANSLATION = self.stack.pop(-1) i18n_disabled = i18n_disabler() def ActivateTranslation(session, config, language, localedir=None): global ACTIVE_TRANSLATION, RECENTLY_TRANSLATED if not language: language = os.getenv('LANG', None) if not localedir: import mailpile.config.paths localedir = mailpile.config.paths.DEFAULT_LOCALE_DIRECTORY() trans = None if (not language) or language[:5].lower() in ('en', 'en_us', 'c'): trans = NullTranslations() elif language: try: trans = translation("mailpile", localedir, [language], codeset="utf-8") except IOError: if session: session.ui.debug('Failed to load language %s' % language) if not trans: trans = translation("mailpile", localedir, codeset='utf-8', fallback=True) if session: session.ui.debug('Failed to configure i18n (%s). ' 'Using fallback.' % language) if trans: with RECENTLY_TRANSLATED_LOCK: RECENTLY_TRANSLATED = [] ACTIVE_TRANSLATION = trans trans.org_gettext = trans.gettext trans.org_ngettext = trans.ngettext trans.gettext = lambda t, g: gettext(g) trans.ngettext = lambda t, s1, s2, n: ngettext(s1, s2, n) trans.set_output_charset("utf-8") if hasattr(config, 'jinja_env'): config.jinja_env.install_gettext_translations(trans, newstyle=True) if session and language and not isinstance(trans, NullTranslations): session.ui.debug(gettext('Loaded language %s') % language) return trans def ListTranslations(config, localedir=None): if not localedir: import mailpile.config.paths localedir = mailpile.config.paths.DEFAULT_LOCALE_DIRECTORY() languages = { 'C': 'English (Mailpile default)' } for lang in os.listdir(localedir): langdir = os.path.join(localedir, lang, 'LC_MESSAGES') if not os.path.exists(os.path.join(langdir, 'mailpile.mo')): continue try: with open(os.path.join(langdir, 'mailpile.po')) as fd: for line in fd.read(8192).splitlines(): line = line.decode('utf-8') if line[1:].startswith('Language-Team: '): languages[lang] = ' '.join([word for word in line[1:-2].split()[1:-1]] ).replace('LANGUAGE', lang) except (IOError, OSError, UnicodeDecodeError): pass return languages ================================================ FILE: mailpile/index/__init__.py ================================================ ================================================ FILE: mailpile/index/base.py ================================================ from __future__ import print_function import copy import json import random import rfc822 import time import traceback from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.index.msginfo import MessageInfoConstants from mailpile.index.search import SearchResultSet from mailpile.mailutils import MBX_ID_LEN from mailpile.mailutils.addresses import AddressHeaderParser from mailpile.mailutils.safe import * from mailpile.util import * class BaseIndex(MessageInfoConstants): MAX_CACHE_ENTRIES = 250 CAN_SEARCH = 'can_search' # Can search message contents CAN_SORT = 'can_sort' # Can sort search results HAS_UNREAD = 'has_unread' # Can filter messages by read/unread HAS_ATTS = 'has_atts' # Can filter messages by attachments or no HAS_TAGS = 'has_tags' # Can filter messages by tags and/or apply tags # What is this Index capable of? A list or set of the above. CAPABILITIES = [] def __init__(self, config): self.config = config self.CACHE = {} self.EMAILS = [] self.EMAIL_IDS = {} ### Known e-mail addresses ############################################# # NOTE: This is all probably a misfeature and should probably go away. def add_email(self, email, name=None, eid=None): if eid is None: eid = len(self.EMAILS) self.EMAILS.append('') self.EMAILS[eid] = '%s (%s)' % (email, name or email) self.EMAIL_IDS[email.lower()] = eid # FIXME: This needs to get written out... return eid def update_email(self, email, name=None, change_name=True): eid = self.EMAIL_IDS.get(email.lower()) if (eid is not None) and not change_name: el = self.EMAILS[eid].split(' ') if len(el) == 2: en = el[1][1:-1] if '@' not in en: name = en return self.add_email(email, name=name, eid=eid) def compact_to_list(self, msg_to): eids = [] for ai in msg_to: email = ai.address eid = self.EMAIL_IDS.get(email.lower()) if eid is None: eid = self.add_email(email, name=ai.fn) elif ai.fn and ai.fn != email: self.update_email(email, name=ai.fn, change_name=False) eids.append(eid) return ','.join([b36(e) for e in set(eids)]) def expand_to_list(self, msg_info, field=None): eids = msg_info[field if (field is not None) else self.MSG_TO] eids = [e for e in eids.strip().split(',') if e] return [self.EMAILS[int(e, 36)] for e in eids] ### Tags & filters ##################################################### def remove_tag(self, session, tid, msg_idxs=None): pass def add_tag(self, session, tid, msg_idxs=None): pass def apply_filters(self, session, fid, msg_idxs=None): pass ### Searching & sorting ################################################ def search(self, session, terms, context=None): return SearchResultSet(self, terms, [], []) def sort_results(self, session, results, sort_order): pass def get_conversation(self, msg_idx=None): return [] ### Loading data: subclasses override these ############################ def get_msg_at_idx_pos_uncached(self, msg_idx): raise IndexError('Unimplemented') def open_mailbox_by_ptr(self, msg_ptr): return self.config.open_mailbox(None, msg_ptr[:MBX_ID_LEN]) ### Loading data: higher level methods ################################# def unique_mbox_ids(self, msg_info): return set([ p[:MBX_ID_LEN] for p in msg_info[self.MSG_PTRS].split(',') if p]) def enumerate_ptrs_mboxes_fds(self, msg_info): for msg_ptr in self._sorted_msg_ptrs(msg_info): mbox = fd = None try: mbox = self.open_mailbox_by_ptr(msg_ptr) fd = mbox.get_file_by_ptr(msg_ptr) except (IOError, OSError, KeyError, ValueError, IndexError): if 'sources' in self.config.sys.debug: traceback.print_exc() print('WARNING: %s not found' % msg_ptr) yield (msg_ptr, mbox, fd) ### ... ################################################################ def _sorted_msg_ptrs(self, msg_info): ptrs = (p.strip() for p in msg_info[self.MSG_PTRS].split(',')) # FIXME: Prefer local data? Prefer some mailbox types? Hmm. # Doing this well would speed things up and ensure the # `message/delete --keep` deduplication works nicely. return sorted([p for p in ptrs if p]) def _encode_msg_id(self, msg_id): """Normalize and hash a message ID for the metadata index""" if '<' in msg_id: new_msg_id = '<%s>' % msg_id.split('<')[1].split('>')[0] if len(new_msg_id) > 2: msg_id = new_msg_id return b64c(sha1b64(msg_id.strip())) def get_msg_id(self, msg, msg_ptr): return self._encode_msg_id(safe_get_msg_id(msg) or msg_ptr) def _message_to_msg_info(self, msg_idx_pos, msg_ptr, msg): msg_mid = b36(msg_idx_pos) msg_to = AddressHeaderParser(msg.get('to')) msg_cc = AddressHeaderParser(msg.get('cc')) msg_cc += AddressHeaderParser(msg.get('bcc')) return [ msg_mid, msg_ptr, # Message PTR self.get_msg_id(msg, msg_ptr), # Message ID b36(safe_message_ts(msg)), # Message timestamp safe_decode_hdr(msg, 'from'), # Message from self.compact_to_list(msg_to), # Compacted to-list self.compact_to_list(msg_cc), # Compacted cc/bcc-list b36(len(msg) // 1024), # Message size safe_decode_hdr(msg, 'subject'), # Subject self.MSG_BODY_LAZY, # Body snippets come later '', # Tags '', # Replies msg_mid] # Thread def get_msg_at_idx_pos(self, msg_idx): try: crv = self.CACHE.get(msg_idx, {}) if 'msg_info' in crv: return crv['msg_info'] if len(self.CACHE) > self.MAX_CACHE_ENTRIES: try: for k in random.sample( self.CACHE.keys(), self.MAX_CACHE_ENTRIES/20): del self.CACHE[k] except KeyError: pass rv = self.get_msg_at_idx_pos_uncached(msg_idx) crv['msg_info'] = rv self.CACHE[msg_idx] = crv return rv except (IndexError, ValueError): return copy.copy(self.BOGUS_METADATA) def set_msg_at_idx_pos(self, msg_idx, msg_info): pass return [] # FIXME @classmethod def get_body(self, msg_info): msg_body = msg_info[self.MSG_BODY] if msg_body.startswith('{'): if msg_body == self.MSG_BODY_LAZY: return {'snippet': _('(unprocessed)'), 'lazy': True} elif msg_body == self.MSG_BODY_GHOST: return {'snippet': _('(ghost)'), 'ghost': True} elif msg_body == self.MSG_BODY_DELETED: return {'snippet': _('(deleted)'), 'deleted': True} try: return json.loads(msg_body) except ValueError: pass return { 'snippet': msg_body } @classmethod def truncate_body_snippet(self, body, max_chars): if 'snippet' in body: delta = len(self.encode_body(body)) - max_chars if delta > 0: body['snippet'] = body['snippet'][:-delta].rsplit(' ', 1)[0] @classmethod def encode_body(self, d, **kwargs): for k, v in kwargs: if v is None: if k in d: del d[k] else: d[k] = v if len(d) == 1 and 'snippet' in d: snippet = d['snippet'] if snippet[:3] in self.MSG_BODY_MAGIC or snippet[:1] != '{': return d['snippet'] return json.dumps(d, indent=None, separators=(',', ':')) @classmethod def set_body(self, msg_info, **kwargs): d = self.get_body(msg_info) msg_info[self.MSG_BODY] = self.encode_body(d, **kwargs) if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/index/mailboxes.py ================================================ from __future__ import print_function import email.parser import json import traceback from mailpile.util import * from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.index.base import BaseIndex from mailpile.index.msginfo import MessageInfoConstants from mailpile.index.search import SearchResultSet from mailpile.mailutils import MBX_ID_LEN class MailboxIndex(BaseIndex): FAKE_MBX_ID = ('Z' * MBX_ID_LEN) def __init__(self, config, mailbox, mbx_mid=None): BaseIndex.__init__(self, config) self.mbx_mid = mbx_mid or self.FAKE_MBX_ID self.mailbox = mailbox self.ptrset = set([]) self.idxmap = [] def get_msg_at_idx_pos_uncached(self, msg_idx_pos): msg_ptr = self.idxmap[msg_idx_pos] msg_raw = self.mailbox.get_file_by_ptr(msg_ptr) message = email.parser.Parser().parse(msg_raw, True) return self._message_to_msg_info(msg_idx_pos, msg_ptr, message) def open_mailbox_by_ptr(self, msg_ptr): if msg_ptr[:MBX_ID_LEN] == self.mbx_mid: return self.mailbox else: return BaseIndex.open_mailbox_by_ptr(self, msg_ptr) def _update_keymap(self): try: mailbox_keys = self.mailbox.keys() mailbox_ptrs = [self.mailbox.get_msg_ptr(self.mbx_mid, i) for i in mailbox_keys] for ptr in mailbox_ptrs: if ptr not in self.ptrset: self.idxmap.append(ptr) self.ptrset = set(mailbox_ptrs) except: traceback.print_exc() def search(self, session, terms, context=None): if not terms or terms == ['all:mail']: self._update_keymap() result = [i for i, ptr in enumerate(self.idxmap) if ptr in self.ptrset] result.reverse() else: print('FIXME! %s: search %s' % (self, terms)) result = [] return SearchResultSet(self, terms, result, []) if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/index/msginfo.py ================================================ from __future__ import print_function from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n class MessageInfoConstants(object): MSG_MID = 0 MSG_PTRS = 1 MSG_ID = 2 MSG_DATE = 3 MSG_FROM = 4 MSG_TO = 5 MSG_CC = 6 MSG_KB = 7 MSG_SUBJECT = 8 MSG_BODY = 9 MSG_TAGS = 10 MSG_REPLIES = 11 MSG_THREAD_MID = 12 MSG_FIELDS_V1 = 11 MSG_FIELDS_V2 = 13 MSG_BODY_LAZY = '{L}' MSG_BODY_GHOST = '{G}' MSG_BODY_DELETED = '{D}' MSG_BODY_UNSCANNED = (MSG_BODY_LAZY, MSG_BODY_GHOST) MSG_BODY_MAGIC = (MSG_BODY_LAZY, MSG_BODY_GHOST, MSG_BODY_DELETED) BOGUS_METADATA = [None, '', None, '0', '(no sender)', '', '', '0', '(not in index)', '', '', '', '-1'] if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/index/search.py ================================================ from __future__ import print_function import time from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n class SearchResultSet: """ Search results! """ def __init__(self, idx, terms, results, exclude): self.terms = set(terms) self._index = idx self.set_results(results, exclude) def set_results(self, results, exclude): self._results = { 'raw': set(results), 'excluded': set(exclude) & set(results) } return self def __len__(self): return len(self._results.get('raw', [])) def as_set(self, order='raw'): return self._results[order] - self._results['excluded'] def excluded(self): return self._results['excluded'] SEARCH_RESULT_CACHE = {} class CachedSearchResultSet(SearchResultSet): """ Cached search result. """ def __init__(self, idx, terms): global SEARCH_RESULT_CACHE self.terms = set(terms) self._index = idx self._results = SEARCH_RESULT_CACHE.get(self._skey(), {}) self._results['_last_used'] = time.time() def _skey(self): return ' '.join(self.terms) def set_results(self, *args): global SEARCH_RESULT_CACHE SearchResultSet.set_results(self, *args) SEARCH_RESULT_CACHE[self._skey()] = self._results self._results['_last_used'] = time.time() return self @classmethod def DropCaches(cls, msg_idxs=None, tags=None): # FIXME: Make this more granular global SEARCH_RESULT_CACHE SEARCH_RESULT_CACHE = {} if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mail_source/__init__.py ================================================ import datetime import os import random import re import thread import threading import traceback import time import mailpile.util import mailpile.vfs from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import * from mailpile.mailutils import FormatMbxId from mailpile.util import * from mailpile.vfs import vfs, FilePath, MailpileVfsBase __all__ = ['local', 'imap', 'pop3'] class BaseMailSource(threading.Thread): """ MailSources take care of managing a group of mailboxes, synchronizing the source with Mailpile's local metadata and/or caches. """ DEFAULT_JITTER = 15 # Fudge factor to tame thundering herds SAVE_STATE_INTERVAL = 3600 # How frequently we pickle our state INTERNAL_ERROR_SLEEP = 900 # Pause time on error, in seconds RESCAN_BATCH_SIZE = 200 # Index at most this many new e-mails at once MAX_PATHS = 2000 # Limit how many directories we scan at once # This is a helper for the events. __classname__ = 'mailpile.mail_source.BaseMailSource' class MailSourceVfs(MailpileVfsBase): """Generic VFS layer for this mail source.""" def __init__(self, config, source, *args, **kwargs): MailpileVfsBase.__init__(self, *args, **kwargs) self.config = config self.source = source self.root = FilePath('/src:%s' % self.source.my_config._key) def _get_mbox_id(self, path): return path[len(self.root.raw_fp)+1:] def Handles(self, path): path = FilePath(path) return (self.root == path or path.raw_fp.startswith(self.root.raw_fp)) def glob_(self, *args, **kwargs): return self.listdir_(*args, **kwargs) def listdir_(self, where, **kwargs): return [m for m in self.source.my_config.mailbox.keys()] def open_(self, fp, *args, **kwargs): raise IOError('Cannot open Mail Source entries (yet)') def abspath_(self, path): if not path.startswith(self.root.raw_fp): path = self.root.join(path).raw_fp if path == self.root: return path try: mbox_id = self._get_mbox_id(path) path = self.config.sys.mailbox[mbox_id] if path.startswith('src:'): return '/%s' % path return path except (ValueError, KeyError, IndexError): raise OSError('Not found: %s' % path) def isdir_(self, fp): return (self.root == fp) def ismailsource_(self, fp): return (self.root == fp) def mailbox_type_(self, fp, config): return False if (fp == self.root) else 'source' # Fixme def getsize_(self, path): return None def display_name_(self, path, config): if (self.root == path): return (self.source.my_config.name or self.source.my_config._key) try: mbox_id = self._get_mbox_id(path) return self.source.my_config.mailbox[mbox_id].name except (ValueError, KeyError, IndexError): raise OSError('Not found: %s' % path) def exists_(self, fp): return ((self.root == fp) or (fp[len(self.root)+1:] in self.source.my_config.mailbox)) def __init__(self, session, my_config): threading.Thread.__init__(self) self.daemon = mailpile.util.TESTING self._lock = MSrcRLock() self.my_config = my_config self.name = my_config.name self.session = session self.alive = None self.event = None self.jitter = self.DEFAULT_JITTER self._state = 'Idle' self._sleeping = None self._interrupt = None self._rescanning = False self._rescan_waiters = [] self._rescan_forced = False self._loop_count = 0 self._last_rescan_count = (0, 0) self._last_rescan_completed = False self._last_rescan_failed = False self._last_saved = time.time() # Saving right away would be silly ms_vfs = self.MailSourceVfs(session.config, self) mailpile.vfs.register_handler(5000, ms_vfs) def __str__(self): rv = ': '.join([threading.Thread.__str__(self), self._state]) if self._sleeping > 0: rv += '(%s)' % self._sleeping return rv def _pfn(self): return 'mail-source.%s' % self.my_config._key def _load_state(self): with self._lock: config, my_config = self.session.config, self.my_config events = list(config.event_log.incomplete(source=self, data_id=my_config._key)) if events: self.event = events[0] if my_config.enabled: self.event.message = _('Starting up') else: self.event.message = _('Disabled') else: self.event = config.event_log.log( source=self, flags=Event.RUNNING, message=_('Starting up'), data={'id': my_config._key}) self.event.data['name'] = my_config.name or _('Mail Source') if 'counters' not in self.event.data: self.event.data['counters'] = {} for c in ('copied_messages', 'indexed_messages', 'unknown_policies'): if c not in self.event.data['counters']: self.event.data['counters'][c] = 0 def _save_state(self): self.session.config.event_log.log_event(self.event) def _save_config(self): self.session.config.save_worker.add_unique_task( self.session, 'Save config', self.session.config.save) def _log_status(self, message, clear_errors=False): # If the user renames our parent_tag, we assume the name change too. self.update_name_to_match_tag() if clear_errors: err = self.event.data.get('connection', {}).get('error', [False]) if err[0]: err[:] = [False, _('Nothing is wrong')] self.event.message = message self.session.config.event_log.log_event(self.event) self.session.ui.mark(message) if 'sources' in self.session.config.sys.debug: self.session.ui.debug('%s: %s' % (self, message)) def open(self): """Open mailboxes or connect to the remote mail source.""" raise NotImplemented('Please override open in %s' % self) def close(self): """Close mailboxes or disconnect from the remote mail source.""" raise NotImplemented('Please override open in %s' % self) def _has_mailbox_changed(self, mbx, state): """For the default sync_mail routine, report if mailbox changed.""" raise NotImplemented('Please override _has_mailbox_changed in %s' % self) def _mark_mailbox_rescanned(self, mbx, state): """For the default sync_mail routine, note mailbox was rescanned.""" raise NotImplemented('Please override _mark_mailbox_rescanned in %s' % self) def _path(self, mbx): if mbx.path.startswith('@'): return self.session.config.sys.mailbox[mbx.path[1:]] else: return mbx.path def _check_interrupt(self, log=True, clear=True): if not self._interrupt: full_path = self.session.config.need_more_disk_space() if full_path is not None: self._interrupt = _('Insufficient free space in %s' ) % full_path if (mailpile.util.QUITTING or self._interrupt or not self.my_config.enabled): if log: self._log_status(_('Interrupted: %s') % (self._interrupt or _('Shutting down')), clear_errors=(not self._interrupt)) if clear: self._interrupt = None return True else: return False def _mailbox_sort_key(self, m): return md5_hex(str(self._loop_count), m.name) def _sorted_mailboxes(self): mailboxes = self.my_config.mailbox.values() mailboxes.sort(key=lambda m: ( 'inbox' in m.name.lower() and 1 or 2, 'sent' in m.name.lower() and 1 or 2, 'spam' in m.name.lower() and 1 or 2, # For training filters! '[Gmail]' in m.name and 2 or 1, # This goes last... self._mailbox_sort_key(m))) return mailboxes def _policy(self, mbx_cfg): policy = mbx_cfg.policy if policy == 'inherit': return self.my_config.discovery.policy return policy def update_name_to_match_tag(self): parent_tag_id = self.my_config.discovery.parent_tag if parent_tag_id and parent_tag_id != '!CREATE': tag = self.session.config.get_tag(parent_tag_id) if tag and self.name != tag.name: self.name = self.my_config.name = tag.name if self.event: self.event.data['name'] = self.name def sync_mail(self): """Iterates through all the mailboxes and scans if necessary.""" config = self.session.config rescanned = messages = errors = 0 self._last_rescan_count = (0, 0) self._last_rescan_completed = False self._last_rescan_failed = False self._interrupt = None batch = min(self._loop_count * 20, self.RESCAN_BATCH_SIZE) errors = rescanned = 0 all_completed = True ostate = self._state if not self._check_interrupt(clear=False): self._state = 'Waiting... (disco)' discovered = self.discover_mailboxes() else: discovered = 0 plan = self._sorted_mailboxes() self.event.data['plan'] = [[m._key, _('Pending'), m.name] for m in plan] event_plan = dict((mp[0], mp) for mp in self.event.data['plan']) if plan and random.randint(0, 20) == 1: random_plan = [m._key for m in random.sample(plan, 1)] else: random_plan = [] for mbx_cfg in plan: if not self._rescan_forced: play_nice_with_threads(weak=True) if self._check_interrupt(clear=False): all_completed = False break try: with self._lock: mbx_key = FormatMbxId(mbx_cfg._key) path = self._path(mbx_cfg) policy = self._policy(mbx_cfg) if (path in ('/dev/null', '', None) or policy in ('ignore', 'unknown')): event_plan[mbx_cfg._key][1] = _('Skipped') continue # Generally speaking, we only rescan if a mailbox looks like # it has changed. However, every once in a while (see logic # around random_mailboxes above) we check anyway just in case # looks are deceiving. state = {} if batch < 1: event_plan[mbx_cfg._key][1] = _('Postponed') elif (self._has_mailbox_changed(mbx_cfg, state) or self._rescan_forced or mbx_cfg.local == '!CREATE' or mbx_cfg._key in random_plan): event_plan[mbx_cfg._key][1] = _('Working ...') this_batch = max(5, int(0.7 * batch)) self._state = 'Waiting... (rescan)' if self._check_interrupt(clear=False): all_completed = False break count = self.rescan_mailbox(mbx_key, mbx_cfg, path, stop_after=this_batch) if count >= 0: self.event.data['counters' ]['indexed_messages'] += count batch -= count this_batch -= count messages += count complete = ((count == 0 or this_batch > 0) and not self._interrupt and not mailpile.util.QUITTING) if complete: rescanned += 1 # If there was a copy, check if it completed cstate = self.event.data.get('copying') or {} if not cstate.get('complete', True): complete = False # If there was a rescan, check if it completed rstate = self.event.data.get('rescan') or {} if not rstate.get('complete', True): complete = False # OK, everything looks complete, mark it! if complete: event_plan[mbx_cfg._key][1] = _('Completed') self._mark_mailbox_rescanned(mbx_cfg, state) else: event_plan[mbx_cfg._key][1] = _('Indexed %d' ) % count all_completed = False if count == 0 and ('sources' in config.sys.debug): time.sleep(60) else: event_plan[mbx_cfg._key][1] = _('Failed') self._last_rescan_failed = True all_completed = False errors += 1 else: event_plan[mbx_cfg._key][1] = _('Unchanged') except (NoSuchMailboxError, IOError, OSError) as e: event_plan[mbx_cfg._key][1] = '%s: %s' % (_('Error'), e) self._last_rescan_failed = True errors += 1 except Exception as e: event_plan[mbx_cfg._key][1] = '%s: %s' % ( _('Internal error'), e) self._rescan_forced = False self._last_rescan_failed = True self._log_status(_('Internal error')) raise self._last_rescan_completed = all_completed self._state = 'Done' status = [] if discovered > 0: status.append(_('Discovered %d mailboxes') % discovered) self._last_rescan_completed = False if rescanned > 0: status.append(_('Processed %d mailboxes') % rescanned) if errors: status.append(_('Failed to process %d') % errors) if not status: status.append(_('No new mail at %s' ) % datetime.datetime.today().strftime('%H:%M')) self._rescan_forced = False self._log_status(', '.join(status)) self._last_rescan_count = (messages, rescanned) self._state = ostate return rescanned def _jitter(self, seconds): return seconds + random.randint(0, self.jitter) def _sleeping_is_ok(self, slept): return True def _sleep(self, seconds): enabled = self.my_config.enabled if self._rescan_forced: seconds = 0 else: play_nice_with_threads() if 'sources' in self.session.config.sys.debug: self.session.ui.debug('Sleeping up to %d seconds...' % seconds) if self._sleeping is None: self._sleeping = seconds while (self.alive and (self._sleeping > 0) and self._sleeping_is_ok(seconds - self._sleeping) and (enabled == self.my_config.enabled) and not mailpile.util.QUITTING): time.sleep(max(0, min(1, self._sleeping))) self._sleeping -= 1 self._sleeping = None return (self.alive and not mailpile.util.QUITTING) def _existing_mailboxes(self): return set(self.session.config.sys.mailbox + [mbx_cfg.local for mbx_cfg in self.my_config.mailbox.values() if mbx_cfg.local]) def _update_unknown_state(self): have_unknown = 0 for mailbox in self.my_config.mailbox.values(): if mailbox.policy == 'unknown': have_unknown += 1 self.event.data['counters']['unknown_policies'] = have_unknown self.event.data['have_unknown'] = (have_unknown > 0) def reset_event_discovery_state(self): for k in ('discovery_error', 'discovery_limit', 'discovery_state'): if k in self.event.data: del self.event.data[k] def set_event_discovery_state(self, state=None, error=None, status=None): self.event.data['discovery_limit'] = ( self.my_config.discovery.max_mailboxes) if state is not None: self.event.data['discovery_state'] = state if error is not None: self.event.data['discovery_error'] = error if status is not None: self._log_status(status) def on_event_discovery_starting(self): ostate, self._state = self._state, 'Discovery' self.reset_event_discovery_state() self.set_event_discovery_state( 'scanning', status=_('Checking for new mailboxes')) return ostate def on_event_discovery_toomany(self): self.set_event_discovery_state( error='toomany', status=_('Too many mailboxes found! Raise limits to continue.')) old_limit = self.my_config.discovery.max_mailboxes for i in range(0, 15): if old_limit == self.my_config.discovery.max_mailboxes: self._sleep(2) else: return True return False def on_event_discovery_done(self, ostate): self.set_event_discovery_state('done') self._state = ostate def discover_mailboxes(self, paths=None): config = self.session.config ostate = self.on_event_discovery_starting() try: existing = self._existing_mailboxes() max_mailboxes = self.my_config.discovery.max_mailboxes max_mailboxes -= len(self.my_config.mailbox) adding = [] paths = [(p.encode('utf-8') if isinstance(p, unicode) else p) for p in (paths or self.my_config.discovery.paths)] paths.sort() while paths: raw_fn = paths.pop(0) if 'sources' in config.sys.debug: self.session.ui.mark(_('Checking for new mailboxes in %s' ) % raw_fn.decode('utf-8')) fn = os.path.normpath(os.path.expanduser(raw_fn)) fn = os.path.abspath(fn) if not os.path.exists(fn): continue is_mailbox = False if (raw_fn not in existing and fn not in existing and fn not in adding): if self.is_mailbox(fn): adding.append(fn) is_mailbox = True if len(adding) and (len(adding) > max_mailboxes): break if os.path.isdir(fn): try: max_paths = self.MAX_PATHS - len(paths) subdirs = [f for f in os.listdir(fn) if f not in ('.', '..')] if len(subdirs) > (max_paths/2): # If we are hitting our limits, randomize. random.shuffle(subdirs) else: # Otherwise, do things in an orderly fashion. subdirs.sort() for f in subdirs[:max_paths/2]: nfn = os.path.join(fn, f) if is_mailbox and f in ('cur', 'new', 'tmp'): pass # Skip Maildir special directories elif (len(paths) <= self.MAX_PATHS and os.path.isdir(nfn)): paths.append(nfn) elif self.is_mailbox(nfn): paths.append(nfn) play_nice_with_threads(weak=True) except OSError: pass # This may have been a bit of work, take a break. play_nice_with_threads() if len(adding) and (len(adding) > max_mailboxes): break if len(adding) and (len(adding) > max_mailboxes): if self.on_event_discovery_toomany(): return self.discover_mailboxes(paths=paths) adding = adding[:max(0, max_mailboxes)] if adding: self.set_event_discovery_state('adding') play_nice_with_threads() new = {} for path in adding: new[config.sys.mailbox.append(path)] = path for mailbox_idx in new.keys(): mbx_cfg = self.take_over_mailbox(mailbox_idx, save=False) if self._policy(mbx_cfg) != 'unknown': del new[mailbox_idx] self._save_config() return len(adding) except: if config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) raise finally: self.on_event_discovery_done(ostate) def _default_policy(self, mbx_cfg): return 'inherit' def take_over_mailbox(self, mailbox_idx, policy=None, create_local=None, save=True, guess_tags=None, apply_tags=None): config = self.session.config disco_cfg = self.my_config.discovery # Stayin' alive! Stayin' alive! with self._lock: mailbox_idx = FormatMbxId(mailbox_idx) self.my_config.mailbox[mailbox_idx] = { 'path': '@%s' % mailbox_idx, 'policy': policy or 'inherit', 'process_new': disco_cfg.process_new, 'local': '!CREATE' if create_local else '', } mbx_cfg = self.my_config.mailbox[mailbox_idx] mbx_cfg.apply_tags.extend(disco_cfg.apply_tags) if apply_tags: mbx_cfg.apply_tags.extend(t for t in apply_tags if t) mbx_cfg.policy = policy or self._default_policy(mbx_cfg) mbx_cfg.name = self._mailbox_name(self._path(mbx_cfg)) if guess_tags is None: guess_tags = disco_cfg.guess_tags if guess_tags: self._guess_tags(mbx_cfg) self._create_primary_tag(mbx_cfg, save=False) self._create_local_mailbox(mbx_cfg, save=False) if save: self._save_config() return mbx_cfg def _guess_tags(self, mbx_cfg): if not mbx_cfg.name: return mbx_cfg.apply_tags = sorted(list( set(mbx_cfg.apply_tags) | self.session.config.guess_tags(mbx_cfg.name))) def _strip_file_extension(self, path): return path.rsplit('.', 1)[0] def _mailbox_path_split(self, path): return ('/' in path) and path.split('/') or path.split('\\') def _mailbox_name(self, path): return self._mailbox_path_split(path)[-1] def _create_local_mailbox(self, mbx_cfg, save=True): config = self.session.config disco_cfg = self.my_config.discovery if mbx_cfg.local and mbx_cfg.local != '!CREATE': if not vfs.exists(mbx_cfg.local): config.flush_mbox_cache(self.session) path, wervd = config.create_local_mailstore(self.session, name=mbx_cfg.local) wervd.is_local = mbx_cfg._key mbx_cfg.local = path if save: self._save_config() elif mbx_cfg.local == '!CREATE' or disco_cfg.local_copy: config.flush_mbox_cache(self.session) path, wervd = config.create_local_mailstore(self.session) wervd.is_local = mbx_cfg._key mbx_cfg.local = path if save: self._save_config() return mbx_cfg def _create_parent_tag(self, save=True): disco_cfg = self.my_config.discovery if disco_cfg.parent_tag: if disco_cfg.parent_tag == '!CREATE': name = (self.my_config.name or (self.my_config.username or '').split('@')[-1] or (disco_cfg.paths and os.path.basename(disco_cfg.paths[0])) or self.my_config._key) if len(name) < 4: name = _('Mail: %s') % name disco_cfg.parent_tag = name if disco_cfg.parent_tag not in self.session.config.tags.keys(): from mailpile.plugins.tags import Slugify disco_cfg.parent_tag = self._create_tag( disco_cfg.parent_tag, use_existing=False, icon='icon-mailsource', slug=Slugify( self.my_config.name, tags=self.session.config.tags), unique=False) if save: self._save_config() return disco_cfg.parent_tag else: return None def _create_primary_tag(self, mbx_cfg, save=True): config = self.session.config if mbx_cfg.primary_tag and (mbx_cfg.primary_tag in config.tags): return # Stayin' alive! Stayin' alive! disco_cfg = self.my_config.discovery if not disco_cfg.create_tag: return # Make sure we have a parent tag, as that maybe useful when creating # tag names or the primary tag itself. parent = self._create_parent_tag(save=False) # We configure the primary_tag with a name, if it doesn't have # one already. if not mbx_cfg.primary_tag: mbx_cfg.primary_tag = self._create_tag_name(self._path(mbx_cfg)) # If we have a policy for this mailbox, we really go and create # tags. The gap here allows the user to edit the primary_tag # proposal before changing the policy from 'unknown'. if self._policy(mbx_cfg) != 'unknown': try: mbx_cfg.primary_tag = self._create_tag( mbx_cfg.primary_tag, use_existing=False, visible=disco_cfg.visible_tags, unique=False, parent=parent) except (ValueError, IndexError): self.session.ui.debug(traceback.format_exc()) if save: self._save_config() BORING_FOLDER_RE = re.compile('(?i)^(home|mail|data|user\S*|[^[:alpha:]]+)$', re.UNICODE) TAGNAME_STRIP_RE = re.compile('[{}\\[\\]]', re.UNICODE) def _path_to_tagname(self, path): # -> tag name """This converts a path to a tag name.""" parts = self._mailbox_path_split(path) parts = [p for p in parts if not re.match(self.BORING_FOLDER_RE, p)] if not parts: return _('Unnamed') tagname = self._strip_file_extension(parts.pop(-1)) while tagname[:1] == '.': tagname = tagname[1:] return re.sub(self.TAGNAME_STRIP_RE, '', tagname.replace('_', ' ')) def _unique_tag_name(self, tagname): # -> unused tag name """Make sure a tagname really is unused, unless we have a parent""" if self.my_config.discovery.parent_tag: return tagname tagnameN, count = tagname, 2 while self.session.config.get_tags(tagnameN): tagnameN = '%s (%s)' % (tagname, count) count += 1 return tagnameN def _create_tag_name(self, path): # -> unique tag name """Convert a path to a unique tag name.""" return self._unique_tag_name(self._path_to_tagname(path)) def _create_tag(self, tag_name_or_id, use_existing=True, unique=False, label=False, visible=True, slug=None, icon=None, parent=None): # -> tag ID if tag_name_or_id in self.session.config.tags: # Short circuit if this is a tag ID for an existing tag return tag_name_or_id else: tag_name = tag_name_or_id tags = self.session.config.get_tags(tag_name) if tags and unique: raise ValueError('Tag name is not unique!') elif len(tags) == 1 and use_existing: tag_id = tags[0]._key else: if slug is None: from mailpile.plugins.tags import Slugify if self.my_config.name: slug = Slugify('/'.join([self.my_config.name, tag_name]), tags=self.session.config.tags) else: slug = Slugify(tag_name, tags=self.session.config.tags) tag_id = self.session.config.tags.append({ 'name': tag_name, 'slug': slug, 'type': 'mailbox', 'parent': parent or '', 'label': label, 'flag_allow_add': False, 'flag_allow_del': False, 'icon': icon or 'icon-tag', 'display': 'tag' if visible else 'archive', }) if parent and visible: self.session.config.tags[parent].display = 'tag' return tag_id def interrupt_rescan(self, reason): self._interrupt = reason or _('Aborted') if self._rescanning: self.session.config.index.interrupt = reason def _process_new(self, mbx_key, mbx_cfg, mbox, msg, msg_metadata_kws, msg_ts, keywords, snippet): # Here subclasses could use mbx_key, mbx_cfg or mbox to grab the # mailbox itself, in case it has metadata (like Maildir). The # default just looks at the Status: headers of the mail itself. return ProcessNew(self.session, msg, msg_metadata_kws, msg_ts, keywords, snippet) def _msg_key_order(self, key): return key def _copy_new_messages(self, mbx_key, mbx_cfg, src, stop_after=-1, scan_args=None, deadline=None): session, config = self.session, self.session.config self.event.data['copying'] = progress = { 'running': True, 'mailbox_id': mbx_key, 'copied_messages': 0, 'copied_bytes': 0, 'deleting': False, 'complete': False} scan_args = scan_args or {} policy = self._policy(mbx_cfg) count = 0 def maybe_delete_from_server(loc, src): # Delete from source, if that's our policy. if policy != 'move': return # Messages we have downloaded are candidates for deletion. downloaded = list(set(src.keys()) & set(loc.source_map.keys())) downloaded.sort(key=self._msg_key_order) # If prefs.deletion_ratio is less than 1.0, leave the most # recent messages on the server (so other clients have a # chance to see them too). # # FIXME: This is a hack. It would be better to check the # timestamps of the messages and always leave on the # server for a fixed, configurable amount of time. # if config.prefs.deletion_ratio < 1.0: random_ratio = random.random() * config.prefs.deletion_ratio cutoff = int(max( random_ratio * len(downloaded), # Jitter. Without this, the last message never gets deleted: random.randint(-10, 1))) else: cutoff = len(downloaded) should = _('Should delete %d messages') % cutoff if 'sources' in config.sys.debug and downloaded: session.ui.debug(should) if config.prefs.allow_deletion: try: for i, key in enumerate(downloaded): if i >= cutoff: break progress['deleting'] = '%d/%d' % (i+1, cutoff) src.remove(key) src.flush() except: # Just ignore errors for now, we'll try again later. if 'sources' in config.sys.debug: session.ui.debug(traceback.format_exc()) else: progress['deleting'] = '. '.join([ _('Deletion is disabled'), should]) try: # Lock the source mailbox while we work with it src.lock() with self._lock: loc = config.open_mailbox(session, mbx_key, prefer_local=True) if src == loc: return count # Perform housekeeping on the source_map, to make sure it does # not grow without bounds or misrepresent things. gone = [] src_keys = set(src.keys()) loc_keys = set(loc.keys()) for key, val in loc.source_map.iteritems(): if (val not in loc_keys) or (key not in src_keys): gone.append(key) for key in gone: del loc.source_map[key] # Figure out what actually needs to be downloaded, log it keys = list(src_keys - set(loc.source_map.keys())) keys.sort(key=self._msg_key_order) progress.update({ 'total': len(src_keys), 'total_local': len(loc_keys), 'uncopied': len(keys), 'batch_size': stop_after if (stop_after > 0) else len(keys)}) # Go download! key_errors = [] for key in reversed(keys): if self._check_interrupt(log=False, clear=False): progress['interrupted'] = True return count session.ui.mark(_('Copying message: %s') % key) progress['copying_src_id'] = key try: mkws = src.get_metadata_keywords(key) data = src.get_bytes(key) except KeyError: progress['key_errors'] = key_errors key_errors.append(key) # Ignore, in case this is a problem with just this # individual message... continue loc_key = loc.add_from_source(key, mkws, data) self.event.data['counters']['copied_messages'] += 1 del progress['copying_src_id'] progress['copied_messages'] += 1 progress['copied_bytes'] += len(data) progress['uncopied'] -= 1 count += 1 # This forks off a scan job to index the message config.index.scan_one_message( session, mbx_key, loc, loc_key, wait=False, msg_data=data, msg_metadata_kws=mkws, **scan_args) stop_after -= 1 if (stop_after == 0) or (deadline and time.time() > deadline): maybe_delete_from_server(loc, src) progress['stopped'] = True return count progress['complete'] = True except IOError: # These just abort the download/read, which we're going to just # take in stride for now. if 'sources' in config.sys.debug: session.ui.debug(traceback.format_exc()) progress['ioerror'] = True except: if 'sources' in config.sys.debug: session.ui.debug(traceback.format_exc()) progress['raised'] = True raise finally: progress['running'] = False src.unlock() maybe_delete_from_server(loc, src) return count def rescan_mailbox(self, mbx_key, mbx_cfg, path, stop_after=None): session, config = self.session, self.session.config with self._lock: if self._rescanning: return -1 self._rescanning = True mailboxes = min(1, len([m for m in self.my_config.mailbox.values() if self._policy(m) not in ('ignore', 'unknown')])) try: ostate = self._state # Set this in case locking fails with self._lock: new_state = 'Rescan(%s, %s)' % (mbx_key, stop_after) ostate, self._state = self._state, new_state apply_tags = mbx_cfg.apply_tags[:] parent = self._create_parent_tag(save=True) if parent: tid = config.get_tag_id(parent) if tid: apply_tags.append(tid) self._create_primary_tag(mbx_cfg) if mbx_cfg.primary_tag: tid = config.get_tag_id(mbx_cfg.primary_tag) if tid: apply_tags.append(tid) with self._lock: mbox = config.open_mailbox(session, mbx_key, prefer_local=False) def process_new(msg, msg_metadata_kws, msg_ts, keywords, snippet): return self._process_new(mbx_key, mbx_cfg, mbox, msg, msg_metadata_kws, msg_ts, keywords, snippet) scan_mailbox_args = { 'process_new': (process_new if mbx_cfg.process_new else False), 'apply_tags': (apply_tags or []), 'stop_after': stop_after, 'event': self.event} copied = count = 0 if mbx_cfg.local or self.my_config.discovery.local_copy: # Note: We copy fewer messages than the batch allows for, # because we might have been aborted on an earlier run and # the rescan may need to catch up. self._create_local_mailbox(mbx_cfg) max_copy = max(min(stop_after, 5), int(0.8 * stop_after)) self._state = '%s: %s' % (new_state, _('Copying')) self._log_status(_('Copying up to %d e-mails from %s' ) % (max_copy, self._mailbox_name(path))) copied = self._copy_new_messages(mbx_key, mbx_cfg, mbox, stop_after=max_copy, scan_args=scan_mailbox_args) count += copied if self._check_interrupt(clear=False): if 'rescan' in self.event.data: self.event.data['rescan']['running'] = False return count self._state = '%s: %s' % (new_state, _('Working')) self._log_status(_('Updating search engine for %s' ) % self._mailbox_name(path)) # Wait for background message scans to complete... config.scan_worker.do(session, 'Wait:%s' % path, lambda: 1) if 'rescans' in self.event.data: self.event.data['rescans'][:-mailboxes] = [] return count + config.index.scan_mailbox(session, mbx_key, mbx_cfg.local or path, config.open_mailbox, force=self._rescan_forced, **scan_mailbox_args) except ValueError: session.ui.debug(traceback.format_exc()) return -1 finally: self._state = ostate self._rescanning = False def open_mailbox(self, mbx_id, fn): # This allows mail sources to override the default mailbox # opening mechanism. Returning false respectfully declines. return None def is_mailbox(self, fn): return False def _summarize_auth(self): return sha1b64(self.my_config.auth_type, '-', self.my_config.username, '-', self.my_config.password) def run(self): play_nice(18) # Reduce priority quite a lot with self.session.config.index_check: self.alive = True self._load_state() _original_session = self.session def sleeptime(): if not self.my_config.enabled: return 24 * 3600 elif self._last_rescan_completed or self._last_rescan_failed: return self.my_config.interval else: return 1 def have_invalid_auth(): conn_err = self.event.data.get('connection', {}).get('error') if conn_err and conn_err[0] in ('oauth2', 'auth'): if ((self._loop_count % 100 != 0) and self._summarize_auth() == conn_err[-1]): self.session.ui.debug('Auth unchanged, doing nothing') return True else: self._log_status(_('Checking new credentials'), clear_errors=True) return False self._loop_count = 0 while self._loop_count == 0 or self._sleep(self._jitter(sleeptime())): self.event.data['enabled'] = self.my_config.enabled self.event.data['profile_id'] = self.my_config.profile if self.my_config.enabled: self.event.flags = Event.RUNNING self._loop_count += 1 else: if self._loop_count > 1: self._log_status(_('Disabled'), clear_errors=True) self._loop_count = 1 self.close() continue self.name = self.my_config.name # In case the config changes self._update_unknown_state() if not self.session.config.index: continue if have_invalid_auth(): continue waiters, self._rescan_waiters = self._rescan_waiters, [] for b, e, s in waiters: try: b.release() except thread.error: pass if s: self.session = s try: if 'traceback' in self.event.data: del self.event.data['traceback'] if self.open(): self.sync_mail() else: self._log_conn_errors() next_save_time = self._last_saved + self.SAVE_STATE_INTERVAL if self.alive and time.time() >= next_save_time: self._save_state() self._check_keepalive() elif self._last_rescan_completed: self._check_keepalive() except: self.event.data['traceback'] = traceback.format_exc() self.session.ui.debug(self.event.data['traceback']) self._log_status(_('Internal error! Sleeping...')) self._sleep(self.INTERNAL_ERROR_SLEEP) finally: for b, e, s in waiters: try: e.release() except thread.error: pass self.session = _original_session self._update_unknown_state() self.close() self._log_status(_('Shutdown'), clear_errors=True) self._save_state() def _check_keepalive(self): if not self.my_config.keepalive: self.close() def _log_conn_errors(self): if 'connection' in self.event.data: cinfo = self.event.data['connection'] if not cinfo.get('live'): err_msg = cinfo.get('error', [None, None])[1] if err_msg: self._log_status(err_msg) def wake_up(self): if 'wakeup' in self.session.config.sys.debug: self.session.ui.debug('%s' % traceback.format_stack()) self._sleeping = -1 def notify_config_changed(self): # FIXME: It would be nice to check if the changes apply to us and # stay asleep otherwise. self.wake_up() def rescan_now(self, session=None, started_callback=None): if not self.my_config.enabled: return (0, 0) begin, end = MSrcLock(), MSrcLock() for l in (begin, end): l.acquire() try: self._rescan_waiters.append((begin, end, session)) self._rescan_forced = True self._interrupt = 'Rescan forced' self.wake_up() while not begin.acquire(False): time.sleep(1) if mailpile.util.QUITTING: return self._last_rescan_count if started_callback: started_callback() while not end.acquire(False): time.sleep(1) if mailpile.util.QUITTING: return self._last_rescan_count return self._last_rescan_count except KeyboardInterrupt: self.interrupt_rescan(_('User aborted')) raise finally: for l in (begin, end): try: l.release() except thread.error: pass def quit(self, join=False): self.interrupt_rescan(_('Shutdown')) self.alive = False self.wake_up() if join and self.isAlive(): self.join() def ProcessNew(session, msg, msg_metadata_kws, msg_ts, keywords, snippet): if False and ('dsn:has' in keywords or 'mdn:has' in keywords): # FIXME: This is a delivery notfication of some sort! # TODO: Figure out what it is telling us, do not mark as "new". return False if ('s:maildir' in msg_metadata_kws # Seen=read, maildir or 'r:maildir' in msg_metadata_kws # Replied, maildir or 'r' in msg.get('status', '').lower() # Read, mbox or 'a' in msg.get('x-status', '').lower()): # PINE, answered return False keywords.update(['%s:in' % tag._key for tag in session.config.get_tags(type='unread')]) return True def MailSource(session, my_config): # FIXME: check the plugin and instanciate the right kind of mail source # for this config section. if my_config.protocol in ('mbox', 'maildir', 'local'): from mailpile.mail_source.local import LocalMailSource return LocalMailSource(session, my_config) elif my_config.protocol in ('imap', 'imap_ssl', 'imap_tls'): from mailpile.mail_source.imap import ImapMailSource return ImapMailSource(session, my_config) elif my_config.protocol in ('pop3', 'pop3_ssl'): from mailpile.mail_source.pop3 import Pop3MailSource return Pop3MailSource(session, my_config) raise ValueError(_('Unknown mail source protocol: %s' ) % my_config.protocol) ================================================ FILE: mailpile/mail_source/imap.py ================================================ from __future__ import print_function # This implements our IMAP mail source. It has been tested against the # following IMAP implementations: # # * Google's GMail (july 2014) # * UW IMAPD (10.1.legacy from 2001) # # # IMAP resonses seen in the wild: # # GMail: # # Message flags: \* \Answered \Flagged \Draft \Deleted \Seen # $Phishing receipt-handled $NotPhishing Junk # # LIST (\HasNoChildren) "/" "Travel" # LIST (\Noselect \HasChildren) "/" "[Gmail]" # # UW IMAPD 10.1: # # Message flags: \* \Answered \Flagged \Deleted \Draft \Seen # # LIST (\NoSelect) "/" 17.03.2002 # LIST (\NoInferiors \Marked) "/" in # LIST (\NoInferiors \UnMarked) "/" todays-junk # # Fastmail.fm: # # Message flags: \Answered \Flagged \Draft \Deleted \Seen $X-ME-Annot-2 # $IsMailingList $IsNotification $HasAttachment $HasTD # # LIST (\Noinferiors \HasNoChildren) "." INBOX # LIST (\HasNoChildren \Archive) "." Archive # LIST (\HasNoChildren \Drafts) "." Drafts # LIST (\HasNoChildren \Junk) "." "Junk Mail" # LIST (\HasNoChildren \Sent) "." "Sent Items" # LIST (\HasNoChildren \Trash) "." Trash # # Mykolab.com: # # # import copy import imaplib import os import re import socket import select import ssl import traceback import time from imaplib import IMAP4_SSL, CRLF from mailbox import Mailbox, Message from urllib import quote, unquote try: import cStringIO as StringIO except ImportError: import StringIO import mailpile.mail_source.imap_utf7 from mailpile.auth import IndirectPassword from mailpile.conn_brokers import Master as ConnBroker from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.index.mailboxes import MailboxIndex from mailpile.mail_source import BaseMailSource from mailpile.mail_source.imap_starttls import IMAP4 from mailpile.mailutils import FormatMbxId, MBX_ID_LEN from mailpile.plugins.oauth import OAuth2 from mailpile.util import * from mailpile.vfs import FilePath # Raise imaplib's default maximum line length to something long and # silly. Some versions of Python ship with this set too low for the # Real World (no matter what the RFCs say). imaplib._MAXLINE = 10 * 1024 * 1024 IMAP_TOKEN = re.compile('("[^"]*"' '|[\\(\\)]' '|[^\\(\\)"\\s]+' '|\\s+)') # These are mailbox names we avoid downloading (by default) BLACKLISTED_MAILBOXES = ( 'drafts', 'chats', '[gmail]/all mail', '[gmail]/important', '[gmail]/starred', 'openpgp_keys') class IMAP_IOError(IOError): pass class WithaBool(object): def __init__(self, v): self.v = v def __nonzero__(self): return self.v def __enter__(self, *a, **kw): return self.v def __exit__(self, *a, **kw): return self.v def _parse_imap(reply): """ This routine will parse common IMAP4 responses into Pythonic data structures. >>> _parse_imap(('OK', ['One (Two (Th ree)) "Four Five"'])) (True, ['One', ['Two', ['Th', 'ree']], 'Four Five']) >>> _parse_imap(('BAD', ['Sorry'])) (False, ['Sorry']) """ if not reply or len(reply) < 2: return False, [] stack = [] pdata = [] for dline in reply[1]: while True: if isinstance(dline, (str, unicode)): m = IMAP_TOKEN.match(dline) else: print('WARNING: Unparsed IMAP response data: %s' % (dline,)) m = None if m: token = m.group(0) dline = dline[len(token):] if token[:1] == '"': pdata.append(token[1:-1]) elif token[:1] == '(': stack.append(pdata) pdata.append([]) pdata = pdata[-1] elif token[:1] == ')': pdata = stack.pop(-1) elif token[:1] not in (' ', '\t', '\n', '\r'): pdata.append(token) else: break return (reply[0].upper() == 'OK'), pdata class ImapMailboxIndex(MailboxIndex): pass class SharedImapConn(threading.Thread): """ This is a wrapper around an imaplib.IMAP4 connection which facilitates sharing of the same conn between different parts of the app. If nobody is using the connection and an IDLE callback is specified, it will switch to IMAP IDLE mode when not otherwise in use. Callers are expected to use the "with sharedconn as conn: ..." syntax. """ def __init__(self, session, conn, idle_mailbox=None, idle_callback=None): threading.Thread.__init__(self) self.daemon = True self.session = session self._lock = MSrcLock() self._conn = conn self._idle_mailbox = idle_mailbox self._idle_callback = idle_callback self._can_idle = False self._idling = False self._selected = None for meth in ('append', 'add', 'authenticate', 'capability', 'fetch', 'noop', 'store', 'expunge', 'close', 'list', 'login', 'logout', 'namespace', 'search', 'uid'): self.__setattr__(meth, self._mk_proxy(meth)) self._update_name() self.start() def _mk_proxy(self, method): def proxy_method(*args, **kwargs): try: safe_assert(self._lock.locked()) if 'mailbox' in kwargs: # We're sharing this connection, so all mailbox methods # need to tell us which mailbox they're operating on. typ, data = self.select(kwargs['mailbox']) if typ.upper() != 'OK': return (typ, data) del kwargs['mailbox'] if 'imap' in self.session.config.sys.debug: self.session.ui.debug('%s(%s %s)' % (method, args, kwargs)) rv = getattr(self._conn, method)(*args, **kwargs) if 'imap' in self.session.config.sys.debug: self.session.ui.debug((' => %s' % (rv,))[:240]) return rv # This is annoyingly repetetive because the imaplib error classes # are subclassed in a strange way. # # In short, we convert imaplib's error, abort and readonly into # a subclass of IOError, so Mailplie's common logic can handle # things gracefully. In the case of abort, we also kill the # connection because it's probably in an unworkable state. # except IMAP4.readonly: if 'imap' in self.session.config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) raise IMAP_IOError('Readonly: %s(%s %s)' % (method, args, kwargs)) except IMAP4.abort: if 'imap' in self.session.config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) self._shutdown() raise IMAP_IOError('Abort: %s(%s %s)' % (method, args, kwargs)) except IMAP4.error: if 'imap' in self.session.config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) raise IMAP_IOError('Error: %s(%s %s)' % (method, args, kwargs)) except: # Default is no-op, just re-raise the exception. This includes # the assertions above; they're logic errors we don't want to # suppress. raise return proxy_method def _shutdown(self): if self._conn: self._conn.shutdown() self._conn = None self._update_name() def close(self): safe_assert(self._lock.locked()) self._selected = None if '(closed)' not in self.name: self.name += ' (closed)' return self._conn.close() def select(self, mailbox='INBOX', readonly=False): # This routine caches the SELECT operations, because we will be # making lots and lots of superfluous ones "just in case" as part # of multiplexing one IMAP connection for multiple mailboxes. safe_assert(self._lock.locked()) if self._selected and self._selected[0] == (mailbox, readonly): return self._selected[1] elif self._selected: try: self._conn.close() except IMAP4.error: # This happens if we haven't previously selected a mailbox pass rv = self._conn.select(mailbox='"%s"' % mailbox, readonly=readonly) if rv[0].upper() == 'OK': info = dict(self._conn.response(f) for f in ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY')) self._selected = ((mailbox, readonly), rv, info) else: info = '(error)' if 'imap' in self.session.config.sys.debug: self.session.ui.debug('select(%s, %s) = %s %s' % (mailbox, readonly, rv, info)) return rv def mailbox_info(self, k, default=None): if not self._selected or not self._selected[2]: return default return self._selected[2].get(k, default) def _update_name(self): name = self._conn and self._conn.host if name: self.name = name elif '(dead)' not in self.name: self.name += ' (dead)' def __enter__(self): if not self._conn: raise IOError('I am dead') self._stop_idling() self._lock.acquire() self._stop_idling() return self def __exit__(self, type, value, traceback): self._start_idling() self._lock.release() def _start_idling(self): self._can_idle = True def _stop_idling(self): self._can_idle = False def _imap_idle(self): if not self._conn: return self._can_idle = True self.select(self._idle_mailbox) if 'imap' in self.session.config.sys.debug: logger = self.session.ui.debug else: logger = lambda x: True def send_line(data): logger('> %s' % data) self._conn.send('%s%s' % (data, CRLF)) def get_line(): data = self._conn._get_line().rstrip() logger('< %s' % data) return data try: send_line('%s IDLE' % self._conn._new_tag()) while self._can_idle and not get_line().startswith('+ '): pass while True: rl = wl = xl = None try: rl, wl, xl = select.select([self._conn.sock], [], [], 1) except socket.error: pass if mailpile.util.QUITTING or not self._can_idle: break elif rl and self._idle_callback(get_line()): self._selected = None break send_line('DONE') # Note: We let the IDLE response drop on the floor, don't care. except (socket.error, OSError) as val: raise self._conn.abort('socket error: %s' % val) def quit(self): self._can_idle = False # Required to avoid deadlock below with self._lock: try: if self._conn and self._conn.file: if self._selected: self._conn.close() self.logout() except (IOError, IMAP4.error, AttributeError): pass self._can_idle = False self._conn = None self._update_name() def run(self): try: idle_counter = 0 while self._conn: # By default, all this does is send a NOOP every 120 seconds # to keep the connection alive (or detect errors). for t in range(0, 120): time.sleep(1 if self._conn else 0) if self._can_idle and self._idle_mailbox: idle_counter += 1 # Once we've been in "can idle" state for 5: IDLE! if idle_counter >= 5: with self as raw_conn: self._imap_idle() else: idle_counter = 0 if self._conn: with self as raw_conn: raw_conn.noop() except: if 'imap' in self.session.config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) finally: self.quit() class SharedImapMailbox(Mailbox): """ This implements a Mailbox view of an IMAP folder. The IMAP connection itself is obtained as a SharedImapConn from a particular mail source. >>> imap = ImapMailSource(session, imap_config) >>> mailbox = SharedImapMailbox(session, imap, conn_cls=_MockImap) >>> #mailbox.add('From: Bjarni\\r\\nBarely a message') """ def __init__(self, session, mail_source, mailbox_path='INBOX', conn_cls=None): self.config = session self.source = mail_source self.editable = False # FIXME: this is technically not true self.path = mailbox_path self.conn_cls = conn_cls self._last_updated = None self._index = None self._factory = None # Unused, for Mailbox compatibility self._broken = None def open_imap(self): return self.source.open(throw=IMAP_IOError, conn_cls=self.conn_cls) def timed_imap(self, *args, **kwargs): return self.source.timed_imap(*args, **kwargs) def last_updated(self): return self._last_updated def _assert(self, test, error): if not test: raise IMAP_IOError(error) def __nonzero__(self): if self._broken is not None: return not self._broken try: with self.open_imap() as imap: ok, data = self.timed_imap(imap.noop, mailbox=self.path) self._broken = False except (IOError, AttributeError): self._broken = True return not self._broken def add(self, message): raise Exception('FIXME: Need to RETURN AN ID.') self._broken = None with self.open_imap() as imap: ok, data = self.timed_imap(imap.append, self.path, message=message) self._last_updated = time.time() self._assert(ok, _('Failed to add message')) self._broken = False def remove(self, key): self._broken = None with self.open_imap() as imap: uidv, uid = (int(k, 36) for k in key.split('.')) ok, data = self.timed_imap(imap.uid, 'STORE', uid, '+FLAGS', '(\Deleted)', mailbox=self.path) self._last_updated = time.time() self._assert(ok, _('Failed to remove message')) self._broken = False def mailbox_info(self, k, default=None): self._broken = None with self.open_imap() as imap: imap.select(self.path) return imap.mailbox_info(k, default=default) self._broken = False def get_info(self, key): self._broken = None with self.open_imap() as imap: uidv, uid = (int(k, 36) for k in key.split('.')) ok, data = self.timed_imap(imap.uid, 'FETCH', uid, # Note: It seems that either python's # imaplib, or our parser cannot # handle dovecot's ENVELOPE # details. So omit that for now. '(RFC822.SIZE FLAGS)', mailbox=self.path) if not ok: raise KeyError(key) self._assert(str(uidv) in imap.mailbox_info('UIDVALIDITY', ['0']), _('Mailbox is out of sync')) info = dict(zip(*[iter(data[1])]*2)) info['UIDVALIDITY'] = uidv info['UID'] = uid self._broken = False return info def get(self, key, _bytes=None): info = self.get_info(key) if 'UID' not in info: raise KeyError(key) # FIXME: This will hard fail to download mail, if our internet # connection averages 8 kbps or worse. Better would be to # adapt the chunk size here to actual network performance. # chunk_size = self.source.timeout * 1024 chunk = 0 msg_data = [] if _bytes and chunk_size > _bytes: chunk_size = _bytes # Some IMAP servers misreport RFC822.SIZE, so we cannot really know # how much data to expect. So we just FETCH chunk until one comes up # short or empty and assume that's it... while chunk >= 0: req = '(BODY.PEEK[]<%d.%d>)' % (chunk * chunk_size, chunk_size) with self.open_imap() as imap: # Note: use the raw method, not the convenient parsed version. typ, data = self.source.timed(imap.uid, 'FETCH', info['UID'], req, mailbox=self.path) self._assert(typ == 'OK', _('Fetching chunk %d failed') % chunk) msg_data.append(data[0][1]) if len(data[0][1]) < chunk_size: chunk = -1 else: chunk += 1 if _bytes and chunk * chunk_size > _bytes: chunk = -1 # FIXME: Should we add a sanity check and complain if we got # significantly less data than expected via. RFC822.SIZE? return info, ''.join(msg_data) def get_message(self, key): info, payload = self.get(key) return Message(payload) def get_bytes(self, key, *args): info, payload = self.get(key, *args) return payload def get_file(self, key): info, payload = self.get(key) return StringIO.StringIO(payload) def iterkeys(self): self._broken = None with self.open_imap() as imap: ok, data = self.timed_imap(imap.uid, 'SEARCH', None, 'ALL', mailbox=self.path) self._assert(ok, _('Failed to list mailbox contents')) validity = imap.mailbox_info('UIDVALIDITY', ['0'])[0] self._broken = False return ('%s.%s' % (b36(int(validity)), b36(int(k))) for k in sorted(data)) def keys(self): return list(self.iterkeys()) def update_toc(self): self._last_updated = time.time() def get_msg_ptr(self, mboxid, key): return '%s%s' % (mboxid, quote(key)) def get_file_by_ptr(self, msg_ptr): return self.get_file(unquote(msg_ptr[MBX_ID_LEN:])) def remove_by_ptr(self, msg_ptr): return self.remove(unquote(msg_ptr[MBX_ID_LEN:])) def get_msg_size(self, key): return long(self.get_info(key).get('RFC822.SIZE', 0)) def get_metadata_keywords(self, key): # Translate common IMAP flags into the maildir vocabulary flags = [f.lower() for f in self.get_info(key).get('FLAGS', '')] mkws = [] for char, flag in (('s', '\\seen'), ('r', '\\answered'), ('d', '\\draft'), ('f', '\\flagged'), ('t', '\\deleted')): if flag in flags: mkws.append('%s:maildir' % char) return mkws def __contains__(self, key): try: self.get_info(key) return True except (KeyError): return False def __len__(self): self._broken = None with self.open_imap() as imap: ok, data = self.timed_imap(imap.noop, mailbox=self.path) return imap.mailbox_info('EXISTS', ['0'])[0] self._broken = False def flush(self): pass def close(self): pass def lock(self): pass def unlock(self): pass def save(self, *args, **kwargs): # SharedImapMailboxes are never pickled to disk. pass def get_index(self, config, mbx_mid=None): if self._index is None: self._index = ImapMailboxIndex(config, self, mbx_mid=mbx_mid) return self._index def __unicode__(self): if self: return _("IMAP: %s") % self.path else: return _("IMAP: %s (not logged in)") % self.path def describe_msg_by_ptr(self, msg_ptr): if self: return _("e-mail with ID %s") % unquote(msg_ptr[MBX_ID_LEN:]) else: return _("remote mailbox is inavailable") def _connect_imap(session, settings, event, conn_cls=None, timeout=30, throw=False, logged_in_cb=None, source=None): def timed(*args, **kwargs): if source is not None: kwargs['unique_thread'] = 'imap/%s' % (source.my_config._key,) return RunTimed(timeout, *args, **kwargs) def timed_imap(*args, **kwargs): if source is not None: kwargs['unique_thread'] = 'imap/%s' % (source.my_config._key,) return _parse_imap(RunTimed(timeout, *args, **kwargs)) conn = None try: # Prepare the data section of our event, for keeping state. for d in ('mailbox_state',): if d not in event.data: event.data[d] = {} ev = event.data['connection'] = { 'live': False, 'error': [False, _('Nothing is wrong')] } # If we are given a conn class, use that - this allows mocks for # testing. if not conn_cls: req_stls = (settings.get('protocol') == 'imap_tls') want_ssl = (settings.get('protocol') == 'imap_ssl') conn_cls = IMAP4_SSL if want_ssl else IMAP4 else: req_stls = want_ssl = False def mkconn(): if want_ssl: need = [ConnBroker.OUTGOING_IMAPS] else: need = [ConnBroker.OUTGOING_IMAP] with ConnBroker.context(need=need): return conn_cls(settings.get('host'), int(settings.get('port'))) conn = timed(mkconn) if hasattr(conn, 'sock'): conn.sock.settimeout(120) conn.debug = ('imaplib' in session.config.sys.debug) and 4 or 0 ok, data = timed_imap(conn.capability) if ok: capabilities = set(' '.join(data).upper().split()) else: capabilities = set() if req_stls or ('STARTTLS' in capabilities and not want_ssl): try: ok, data = timed_imap(conn.starttls) if ok: # Fetch capabilities again after STARTTLS ok, data = timed_imap(conn.capability) capabilities = set(' '.join(data).upper().split()) # Update the protocol to avoid getting downgraded later if settings.get('protocol', '') != 'imap_ssl': settings['protocol'] = 'imap_tls' except (IMAP4.error, IOError, socket.error): ok = False if not ok: ev['error'] = [ 'tls', _('Failed to make a secure TLS connection'), '%s:%s' % (settings.get('host'), settings.get('port'))] if throw: raise throw(ev['error'][1]) return WithaBool(False) username = password = "" try: error_type = 'login' error_msg = _('IMAP Login error') auth_error_type = auth_error_msg = None username = settings.get('username', '').encode('utf-8') password = IndirectPassword( session.config, settings.get('password', '') ).encode('utf-8') if (settings.get('auth_type', '').lower() == 'oauth2' and 'AUTH=XOAUTH2' in capabilities): auth_error_type = 'oauth2' auth_error_msg = _('Access denied by mail server') token_info = OAuth2.GetFreshTokenInfo(session, username) if not (username and token_info and token_info.access_token): raise ValueError("Missing configuration") ok, data = timed_imap( conn.authenticate, 'XOAUTH2', lambda challenge: OAuth2.XOAuth2Response(username, token_info)) if not ok: token_info.access_token = '' else: auth_error_type = 'auth' auth_error_msg = _('Invalid username or password') ok, data = timed_imap(conn.login, username, password) except IMAP4.error as save_error: error_str = save_error.__str__() ok, data = False, error_str if auth_error_type: # Is this a password error or some other kind of error? # If the response contains any RFC5530 response code # check for an RFC5530 auth error code, otherwise check for # "password" (case independent) in the response string. if (re.search('\[AUTHENTICATIONFAILED\]|' '\[AUTHORIZATIONFAILED\]|' '\[EXPIRED\]', error_str) or (not re.search('\[([a-zA-Z]*)\]', error_str) and re.search('(?i)password', error_str) ) ): error_type = auth_error_type error_msg = auth_error_msg except (UnicodeDecodeError, ValueError): ok, data = False, None if not ok: auth_summary = '' if source is not None: auth_summary = source._summarize_auth() ev['error'] = [error_type, error_msg, username, auth_summary] if throw: raise throw(ev['error'][1]) return WithaBool(False) if logged_in_cb is not None: logged_in_cb(conn, ev, capabilities) return conn except TimedOut: if 'imap' in session.config.sys.debug: session.ui.debug('%s' % traceback.format_exc()) ev['error'] = ['timeout', _('Connection timed out')] except (ssl.CertificateError, ssl.SSLError): if 'imap' in session.config.sys.debug: session.ui.debug('%s' % traceback.format_exc()) ev['error'] = ['tls', _('Failed to make a secure TLS connection'), '%s:%s' % (settings.get('host'), settings.get('port'))] except (IMAP_IOError, IMAP4.error): if 'imap' in session.config.sys.debug: session.ui.debug('%s' % traceback.format_exc()) ev['error'] = ['protocol', _('An IMAP protocol error occurred')] except (IOError, AttributeError, socket.error): if 'imap' in session.config.sys.debug: session.ui.debug('%s' % traceback.format_exc()) ev['error'] = ['network', _('A network error occurred')] try: if conn: # Close the socket directly, in the hopes this will boot # any timed-out operations out of a hung state. conn.socket().shutdown(socket.SHUT_RDWR) conn.file.close() except (AttributeError, IOError, socket.error): pass if throw: raise throw(ev['error']) return None class ImapMailSource(BaseMailSource): """ This is a mail source that connects to an IMAP server. A single connection is made to the IMAP server, which is then shared between the ImapMailSource job and individual mailbox instances. """ # This is a helper for the events. __classname__ = 'mailpile.mail_source.imap.ImapMailSource' TIMEOUT_INITIAL = 60 TIMEOUT_LIVE = 120 CONN_ERRORS = (IOError, IMAP_IOError, IMAP4.error, TimedOut) class MailSourceVfs(BaseMailSource.MailSourceVfs): """Expose the IMAP tree to the VFS layer.""" def _imap_path(self, path): if path[:1] == '/': path = path[1:] return path[len(self.root.raw_fp):] def _imap(self, *args, **kwargs): return self.source.timed_imap(*args, **kwargs) def listdir_(self, where, **kwargs): results = [] path = self._imap_path(where) prefix, pathsep = self.source._namespace_info(path) with self.source.open() as conn: if not conn: raise socket.error(_('Not connected to IMAP server.')) if path: ok, data = self._imap(conn.list, path + pathsep, '%') else: ok, data = self._imap(conn.list, '', '%') while ok and len(data) >= 3: (flags, sep, path), data[:3] = data[:3], [] if path.lower() not in BLACKLISTED_MAILBOXES: flags = [f.lower() for f in flags] self.source._cache_flags(path, flags) results.append('/' + self.source._fmt_path(path)) return results def getflags_(self, fp, cfg): if self.root == fp: return BaseMailSource.MailSourceVfs.getflags_(self, fp, cfg) flags = [flag.lower().replace('\\', '') for flag in self.source._cache_flags(self._imap_path(fp)) or []] if not ('hasnochildren' in flags or 'noinferiors' in flags): flags.append('Directory') if not ('noselect' in flags): flags.append('Mailbox') return flags def abspath_(self, fp): return fp def display_name_(self, fp, config): return FilePath(fp).display_basename() def isdir_(self, fp): if self.root == fp: return True flags = self.source._cache_flags(self._imap_path(fp)) or [] return not ('hasnochildren' in flags or 'noinferiors' in flags) def getsize_(self, path): return None def __init__(self, *args, **kwargs): BaseMailSource.__init__(self, *args, **kwargs) self.timeout = self.TIMEOUT_INITIAL self.last_op = 0 self.watching = -1 self.capabilities = set() self.logged_in_at = None self.namespaces = {'private': []} self.flag_cache = {} self.conn = None self.conn_id = '' @classmethod def Tester(cls, conn_cls, *args, **kwargs): tcls = cls(*args, **kwargs) return tcls.open(conn_cls=conn_cls) and tcls or False def timed(self, *args, **kwargs): return RunTimed(self.timeout, *args, **kwargs) def timed_imap(self, *args, **kwargs): return _parse_imap(RunTimed(self.timeout, *args, **kwargs)) def _conn_id(self): def e(s): try: return unicode(s).encode('utf-8') except UnicodeDecodeError: return unicode(s).encode('utf-8', 'replace') return md5_hex('\n'.join([e(self.my_config[k]) for k in ('host', 'port', 'password', 'username')])) def close(self): with self._lock: if self.conn: self.event.data['connection'] = { 'live': False, 'error': [False, _('Nothing is wrong')] } self.conn.quit() self.conn = None def open(self, conn_cls=None, throw=False): conn = self.conn conn_id = self._conn_id() if conn: try: with conn as c: now = time.time() if (conn_id == self.conn_id and (now < self.last_op + 120 or self.timed(c.noop)[0] == 'OK')): # Make the timeout longer, so we don't drop things # on every hiccup and so downloads will be more # efficient (chunk size relates to timeout). self.timeout = self.TIMEOUT_LIVE if now >= self.last_op + 120: self.last_op = now return conn except self.CONN_ERRORS + (AttributeError, ): pass with self._lock: if self.conn == conn: self.conn = None conn.quit() my_config = self.my_config # This facilitates testing, event should already exist in real life. if self.event: event = self.event else: event = Event(source=self, flags=Event.RUNNING, data={}) def logged_in_cb(conn, ev, capabilities): with self._lock: if self.conn is not None: raise IOError('Woah, we lost a race.') self.capabilities = capabilities if 'NAMESPACE' in capabilities: ok, data = self.timed_imap(conn.namespace) if ok: prv, oth, shr = data self.namespaces = { 'private': prv if (prv != 'NIL') else [], 'others': oth if (oth != 'NIL') else [], 'shared': shr if (shr != 'NIL') else [] } if 'IDLE' in capabilities: self.conn = SharedImapConn( self.session, conn, idle_mailbox='INBOX', idle_callback=self._idle_callback) else: self.conn = SharedImapConn(self.session, conn) if self.event: self._log_status(_('Connected to IMAP server %s' ) % my_config.host) if 'imap' in self.session.config.sys.debug: self.session.ui.debug('CONNECTED %s' % self.conn) self.session.ui.debug('CAPABILITIES %s' % self.capabilities) self.session.ui.debug('NAMESPACES %s' % self.namespaces) self.conn_id = conn_id ev['live'] = True conn = _connect_imap(self.session, self.my_config, event, conn_cls=conn_cls, timeout=self.timeout, throw=throw, logged_in_cb=logged_in_cb, source=self) if conn: self.logged_in_at = time.time() return self.conn else: return WithaBool(False) def _idle_callback(self, data): if 'EXISTS' in data: # Stop sleeping and check for mail self.wake_up() return True else: return False def _check_keepalive(self): alive_for = time.time() - (self.logged_in_at or time.time()) if (not self.my_config.keepalive) or alive_for > (12 * 3600): if ('IDLE' not in self.capabilities or alive_for > self.my_config.interval): self.close() def open_mailbox(self, mbx_id, mfn): try: proto_me, path = mfn.split('/', 1) if proto_me.startswith('src:%s' % self.my_config._key): return SharedImapMailbox(self.session, self, mailbox_path=path) except ValueError: pass return None def _get_mbx_id_and_mfn(self, mbx_cfg): mbx_id = FormatMbxId(mbx_cfg._key) return mbx_id, self.session.config.sys.mailbox[mbx_id] def _has_mailbox_changed(self, mbx_cfg, state): shared_mbox = self.open_mailbox(*self._get_mbx_id_and_mfn(mbx_cfg)) uv = state['uv'] = shared_mbox.mailbox_info('UIDVALIDITY', ['0'])[0] ex = state['ex'] = shared_mbox.mailbox_info('EXISTS', ['0'])[0] uvex = '%s/%s' % (uv, ex) if uvex == '0/0': return True return (uvex != self.event.data.get('mailbox_state', {}).get(mbx_cfg._key)) def _mark_mailbox_rescanned(self, mbx, state): uvex = '%s/%s' % (state['uv'], state['ex']) if 'mailbox_state' in self.event.data: self.event.data['mailbox_state'][mbx._key] = uvex else: self.event.data['mailbox_state'] = {mbx._key: uvex} def _namespace_info(self, path): for which, nslist in self.namespaces.iteritems(): for prefix, pathsep in nslist: if path.startswith(prefix): return prefix, pathsep or '/' # This is a hack for older servers that don't do NAMESPACE if path.startswith('INBOX.'): return 'INBOX', '.' return '', '/' def _default_policy(self, mbx_cfg): if self._mailbox_path(self._path(mbx_cfg) ).lower() in BLACKLISTED_MAILBOXES: return 'ignore' else: return 'inherit' def _sorted_mailboxes(self): # This allows changes to BLACKLISTED_MAILBOXES to have an effect # even if peoples' configs say otherwise. return [ m for m in BaseMailSource._sorted_mailboxes(self) if m.name.lower() not in BLACKLISTED_MAILBOXES] def _msg_key_order(self, key): return [int(k, 36) for k in key.split('.')] def _strip_file_extension(self, mbx_path): return mbx_path # Yes, a no-op :) def _decode_path(self, path): try: return path.decode('imap4-utf-7') except: return path def _mailbox_path(self, mbx_path): # len('src:/') = 5 return str(mbx_path[(5 + len(self.my_config._key)):]) def _mailbox_path_split(self, mbx_path): path = self._mailbox_path(mbx_path) prefix, pathsep = self._namespace_info(path) return [self._decode_path(p) for p in path.split(pathsep)] def _mailbox_name(self, mbx_path): path = self._mailbox_path(mbx_path) prefix, pathsep = self._namespace_info(path) return self._decode_path(path[len(prefix):]) def _fmt_path(self, path): return 'src:%s/%s' % (self.my_config._key, path) def _fix_empty_discovery_path_bug(self): if self.my_config.discovery.policy not in ('unknown', 'ignore'): if not self.my_config.discovery.paths: self.my_config.discovery.paths.append('/') def discover_mailboxes(self, paths=None): config = self.session.config ostate = self.on_event_discovery_starting() self._fix_empty_discovery_path_bug() try: paths = copy.copy(paths or self.my_config.discovery.paths) max_mailboxes = self.my_config.discovery.max_mailboxes mailbox_count = len(self.my_config.mailbox) existing = self._existing_mailboxes() mailboxes = [] with self.open() as raw_conn: for p in paths: mailboxes += self._walk_mailbox_path(raw_conn, str(p)) discovered = [mbx for mbx in mailboxes if mbx not in existing] if discovered and (len(discovered) > max_mailboxes - mailbox_count): discovered = discovered[:max(0, max_mailboxes - mailbox_count)] if self.on_event_discovery_toomany(): return self.discover_mailboxes(paths=paths) self.set_event_discovery_state('adding') for path in discovered: idx = config.sys.mailbox.append(path) mbx = self.take_over_mailbox(idx) return len(discovered) except: if config.sys.debug: self.session.ui.debug('%s' % traceback.format_exc()) raise finally: self.on_event_discovery_done(ostate) def _cache_flags(self, path, flags=None): path = self._fmt_path(path) if flags is not None: self.flag_cache[path] = flags return self.flag_cache.get(path) def _walk_mailbox_path(self, conn, prefix): """ Walks the IMAP path recursively and returns a list of all found mailboxes. """ mboxes = [] subtrees = [] # We go well over the maximum here, so the calling code can detect # detect that we want to go over the limits and can ask the user # whether that's OK. max_mailboxes = 5 + (2 * self.my_config.discovery.max_mailboxes) if prefix == '/': prefix = '' try: ok, data = self.timed_imap(conn.list, prefix, '%') while ok and len(data) >= 3: (flags, sep, path), data[:3] = data[:3], [] flags = [f.lower() for f in flags] if '\\noselect' not in flags: if path.lower() not in BLACKLISTED_MAILBOXES: # We cache the flags for this mailbox, they may tell # use useful things about what kind of mailbox it is. self._cache_flags(path, flags) mboxes.append(self._fmt_path(path)) if '\\haschildren' in flags: subtrees.append('%s%s' % (path, sep)) if len(mboxes) > max_mailboxes: break for path in subtrees: if len(mboxes) <= max_mailboxes: mboxes.extend(self._walk_mailbox_path(conn, path)) except self.CONN_ERRORS: pass finally: return mboxes def quit(self, *args, **kwargs): if self.conn: self.conn.quit() return BaseMailSource.quit(self, *args, **kwargs) def TestImapSettings(session, settings, event, timeout=ImapMailSource.TIMEOUT_INITIAL): conn = _connect_imap(session, settings, event, timeout=timeout) if conn: try: conn.socket().shutdown(socket.SHUT_RDWR) conn.file.close() except (IOError, OSError, socket.error): pass return True return False ##[ Test code follows ]####################################################### class _MockImap(object): """ Base mock that pretends to be an imaplib IMAP connection. >>> imap = ImapMailSource(session, imap_config) >>> imap.open(conn_cls=_MockImap) >>> sorted(imap.capabilities) ['IMAP4REV1', 'X-MAGIC-BEANS'] """ DEFAULT_RESULTS = { 'append': ('OK', []), 'capability': ('OK', ['X-MAGIC-BEANS', 'IMAP4rev1']), 'list': ('OK', []), 'login': ('OK', ['"Welcome, human"']), 'noop': ('OK', []), 'select': ('OK', []), } RESULTS = {} def __init__(self, *args, **kwargs): self.host = 'mock' def mkcmd(rval): def cmd(*args, **kwargs): return rval return cmd for cmd, rval in dict_merge(self.DEFAULT_RESULTS, self.RESULTS ).iteritems(): self.__setattr__(cmd, mkcmd(rval)) def __getattr__(self, attr): return self.__getattribute__(attr) class _Mocks(object): """ A bunch of IMAP test classes for testing various configurations. >>> ImapMailSource.Tester(_Mocks.NoDns, session, imap_config) False >>> ImapMailSource.Tester(_Mocks.BadLogin, session, imap_config) False """ class NoDns(_MockImap): def __init__(self, *args, **kwargs): raise socket.gaierror('Oops') class BadLogin(_MockImap): RESULTS = {'login': ('BAD', ['"Sorry dude"'])} if __name__ == "__main__": import doctest import sys import mailpile.config.defaults import mailpile.config.manager import mailpile.ui rules = mailpile.config.defaults.CONFIG_RULES config = mailpile.config.manager.ConfigManager(rules=rules) config.sources.imap = { 'protocol': 'imap_ssl', 'host': 'imap.gmail.com', 'port': 993, 'username': 'nobody', 'password': 'nowhere' } session = mailpile.ui.Session(config) results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={'session': session, 'imap_config': config.sources.imap}) print('%s' % (results, )) if results.failed: sys.exit(1) args = sys.argv[1:] if args: session.config.sys.debug = 'imap' username, password = args.pop(0), args.pop(0) config.sources.imap.username = username config.sources.imap.password = password imap = ImapMailSource(session, config.sources.imap) with imap.open(throw=IMAP_IOError) as conn: print('%s' % (conn.list(), )) mbx = SharedImapMailbox(config, imap, mailbox_path='INBOX') print('%s' % list(mbx.iterkeys())) for key in args: info, payload = mbx.get(key) print('%s(%d bytes) = %s\n%s' % (mbx.get_msg_ptr('0000', key), mbx.get_msg_size(key), info, payload)) ================================================ FILE: mailpile/mail_source/imap_starttls.py ================================================ import imaplib import ssl Commands = { 'STARTTLS': ('NONAUTH') } imaplib.Commands.update(Commands) class IMAP4(imaplib.IMAP4, object): # This is a bugfix for imaplib's default readline method. It is # identical except it raises abort() instead of error() as the # internal state will certainly be broken. # # Note: This would be the function to change if we want to do away # with the line-length limits altogether. # def readline(self): """Read line from remote.""" line = self.file.readline(imaplib._MAXLINE + 1) if len(line) > imaplib._MAXLINE: raise self.abort("got more than %d bytes" % imaplib._MAXLINE) return line def starttls(self, keyfile = None, certfile = None): typ, data = self._simple_command('STARTTLS') if typ != 'OK': raise self.error('no STARTTLS') self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) self.file = self.sock.makefile('rb') return typ, data ================================================ FILE: mailpile/mail_source/imap_utf7.py ================================================ # -*- coding: utf-8- -*- # from: http://piao-tech.blogspot.no/2010/03/get-offlineimap-working-with-non-ascii.html#resources import binascii import codecs # encoding def modified_base64 (s): s = s.encode('utf-16be') return binascii.b2a_base64(s).rstrip('\n=').replace('/', ',') def doB64(_in, r): if _in: r.append('&%s-' % modified_base64(''.join(_in))) del _in[:] def encoder(s): r = [] _in = [] for c in s: ordC = ord(c) if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e: doB64(_in, r) r.append (c) elif c == '&': doB64(_in, r) r.append ('&-') else: _in.append(c) doB64(_in, r) return (str(''.join(r)), len(s)) # decoding def modified_unbase64(s): b = binascii.a2b_base64(s.replace(',', '/') + '===') return unicode (b, 'utf-16be') def decoder (s): r = [] decode = [] for c in s: if c == '&' and not decode: decode.append ('&') elif c == '-' and decode: if len(decode) == 1: r.append('&') else: r.append(modified_unbase64(''.join(decode[1:]))) decode = [] elif decode: decode.append(c) else: r.append(c) if decode: r.append(modified_unbase64(''.join(decode[1:]))) bin_str = ''.join(r) return (bin_str, len(s)) class StreamReader (codecs.StreamReader): def decode (self, s, errors='strict'): return decoder(s) class StreamWriter (codecs.StreamWriter): def decode (self, s, errors='strict'): return encoder(s) def imap4_utf_7(name): if name == 'imap4-utf-7': return (encoder, decoder, StreamReader, StreamWriter) codecs.register(imap4_utf_7) ================================================ FILE: mailpile/mail_source/local.py ================================================ import time import os from mailpile.mail_source import BaseMailSource from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.vfs import FilePath class LocalMailSource(BaseMailSource): """ This is a mail source that watches over one or more local mailboxes. """ # This is a helper for the events. __classname__ = 'mailpile.mail_source.local.LocalMailSource' def __init__(self, *args, **kwargs): BaseMailSource.__init__(self, *args, **kwargs) if not self.my_config.name: self.my_config.name = _('Local mail') self.recently_changed = [] self.my_config.protocol = 'local' # We may be upgrading an old # mbox or maildir source. self.watching = -1 def _sleeping_is_ok(self, slept): if slept > 5: # # If any of the most recently changed mailboxes has changed # again, cut our sleeps short after 5 seconds. By basing this # on recently changed mailboxes, we don't need to explicitly # ask the user which mailbox(es) are being used as Inboxes. # # The number 10 should be "big enough", without us going # nuts and scanning a gazillion mailboxes every second. # if len(self.recently_changed) > 10: self.recently_changed = self.recently_changed[-10:] for mbx in self.recently_changed: if self._has_mailbox_changed(mbx, {}): return False return True def close(self): pass def open(self): with self._lock: mailboxes = self.my_config.mailbox.values() if self.watching == len(mailboxes): return True else: self.watching = len(mailboxes) # Prepare the data section of our event, for keeping state. for d in ('mailbox_state', ): if d not in self.event.data: self.event.data[d] = {} self._log_status(_('Watching %d mbox mailboxes') % self.watching) return True def _get_macmaildir_data(self, path): ds = [d for d in os.listdir(path) if not d.startswith('.') and os.path.isdir(os.path.join(path, d))] return (len(ds) == 1) and os.path.join(path, ds[0], 'Data') def _data_paths(self, mbx): mbx_path = FilePath(self._path(mbx)).raw_fp if os.path.exists(mbx_path): yield mbx_path if os.path.isdir(mbx_path): # Maildir, WERVD for s in ('cur', 'new', 'tmp', 'wervd.ver'): sub_path = os.path.join(mbx_path, s) if os.path.exists(sub_path): yield sub_path # Mac Maildir sub_path = self._get_macmaildir_data(mbx_path) if sub_path: yield sub_path def _mailbox_sort_key(self, mbx): # Sort mailboxes so the most recently modified get scanned first. mt = 0 for p in self._data_paths(mbx): try: mt = max(mt, os.path.getmtime(p)) except (OSError, IOError): pass if mt: return '%20.20d' % (0x10000000000 - long(mt)) else: return BaseMailSource._mailbox_sort_key(self, mbx) def _has_mailbox_changed(self, mbx, state): mtszs = [] for p in self._data_paths(mbx): try: mt = long(os.path.getmtime(p)) sz = long(os.path.getsize(p)) mtszs.append('%s/%s' % (mt, sz)) except (OSError, IOError): pass if not mtszs: # Try to rescan even if the above fails for some reason mt = sz = (int(time.time()) // 7200) mtszs = ['%s/%s' % (mt, sz)] mtsz = state['mtsz'] = ','.join(mtszs) if (mtsz != self.event.data.get('mailbox_state', {}).get(mbx._key)): while mbx in self.recently_changed: self.recently_changed.remove(mbx) self.recently_changed.append(mbx) return True else: return False def _mark_mailbox_rescanned(self, mbx, state): if 'mailbox_state' in self.event.data: self.event.data['mailbox_state'][mbx._key] = state['mtsz'] else: self.event.data['mailbox_state'] = {mbx._key: state['mtsz']} def _is_mbox(self, fn): try: with open(fn, 'rb') as fd: data = fd.read(2048) # No point reading less... if data.startswith('From '): # OK, this might be an mbox! Let's check if the first # few lines look like RFC2822 headers... headcount = 0 for line in data.splitlines(True)[1:]: if (headcount > 2) and line in ('\n', '\r\n'): return True if line[-1:] == '\n' and line[:1] not in (' ', '\t'): parts = line.split(':') if (len(parts) < 2 or ' ' in parts[0] or '\t' in parts[0]): return False headcount += 1 return (headcount > 2) except (IOError, OSError): pass return False def _is_maildir(self, fn): if not os.path.isdir(fn): return False for sub in ('cur', 'new', 'tmp'): subdir = os.path.join(fn, sub) if not os.path.exists(subdir) or not os.path.isdir(subdir): return False return True def _is_macmaildir(self, path): infoplist = os.path.join(path, 'Info.plist') if not os.path.isdir(path) or not os.path.exists(infoplist): return False data = self._get_macmaildir_data(path) return data and os.path.isdir(data) def is_mailbox(self, fn): fn = FilePath(fn).raw_fp return (self._is_maildir(fn) or self._is_macmaildir(fn) or self._is_mbox(fn)) ================================================ FILE: mailpile/mail_source/pop3.py ================================================ import os import ssl import traceback from mailpile.auth import IndirectPassword from mailpile.conn_brokers import Master as ConnBroker from mailpile.mail_source import BaseMailSource from mailpile.mailboxes import pop3 from mailpile.mailutils import FormatMbxId, MBX_ID_LEN from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.util import * # We use this to enable "recent mode" on GMail accounts by default. GMAIL_TLDS = ('gmail.com', 'googlemail.com') def _open_pop3_mailbox(session, event, host, port, username, password, auth_type, protocol, debug, throw=False): cev = event.data['connection'] = { 'live': False, 'error': [False, _('Nothing is wrong')] } try: # FIXME: Nothing actually adds gmail or gmail-full to the protocol # yet, so we're stuck in recent mode only for now. if (username.lower().split('@')[-1] in GMAIL_TLDS or 'gmail' in protocol): if 'gmail-full' not in protocol: username = 'recent:%s' % username if 'ssl' in protocol: need = [ConnBroker.OUTGOING_POP3S] else: need = [ConnBroker.OUTGOING_POP3] with ConnBroker.context(need=need): return pop3.MailpileMailbox(host, port=port, user=username, password=password, auth_type=auth_type, use_ssl=('ssl' in protocol), session=session, debug=debug) except AccessError: cev['error'] = ['auth', _('Invalid username or password'), username, sha1b64(password)] except (ssl.CertificateError, ssl.SSLError): cev['error'] = ['tls', _('Failed to make a secure TLS connection'), '%s:%s' % (host, port)] except (IOError, OSError): cev['error'] = ['network', _('A network error occurred')] event.data['traceback'] = traceback.format_exc() if throw: raise throw(cev['error'][1]) return None class POP3_IOError(IOError): pass class Pop3MailSource(BaseMailSource): """ This is a mail source that watches over one or more POP3 mailboxes. """ # This is a helper for the events. __classname__ = 'mailpile.mail_source.pop3.Pop3MailSource' def __init__(self, *args, **kwargs): BaseMailSource.__init__(self, *args, **kwargs) self.watching = -1 def close(self): mbx = self.my_config.mailbox.values()[0] if mbx: pop3 = self.session.config.open_mailbox(self.session, FormatMbxId(mbx._key), prefer_local=False, from_cache=True) if pop3: pop3.close() def _sleep(self, *args, **kwargs): self.close() return BaseMailSource._sleep(self, *args, **kwargs) def open(self): with self._lock: mailboxes = self.my_config.mailbox.values() if self.watching == len(mailboxes): return True else: self.watching = len(mailboxes) for d in ('mailbox_state', ): if d not in self.event.data: self.event.data[d] = {} self.event.data['connection'] = { 'live': False, 'error': [False, _('Nothing is wrong')] } self._log_status(_('Watching %d POP3 mailboxes') % self.watching) return True def _has_mailbox_changed(self, mbx, state): pop3 = self.session.config.open_mailbox(self.session, FormatMbxId(mbx._key), prefer_local=False) state['stat'] = stat = '%s' % (pop3.stat(), ) return (self.event.data.get('mailbox_state', {}).get(mbx._key) != stat) def _mark_mailbox_rescanned(self, mbx, state): if 'mailbox_state' in self.event.data: self.event.data['mailbox_state'][mbx._key] = state['stat'] else: self.event.data['mailbox_state'] = {mbx._key: state['stat']} def _fmt_path(self): return 'src:%s' % (self.my_config._key,) def open_mailbox(self, mbx_id, mfn): my_cfg = self.my_config if 'src:' in mfn[:5] and FormatMbxId(mbx_id) in my_cfg.mailbox: debug = ('pop3' in self.session.config.sys.debug) and 99 or 0 password = IndirectPassword(self.session.config, my_cfg.password) return _open_pop3_mailbox(self.session, self.event, my_cfg.host, my_cfg.port, my_cfg.username, password, my_cfg.auth_type, my_cfg.protocol, debug, throw=POP3_IOError) return None def discover_mailboxes(self, paths=None): config = self.session.config existing = self._existing_mailboxes() if self._fmt_path() not in existing: idx = config.sys.mailbox.append(self._fmt_path()) self.take_over_mailbox(idx) return 1 return 0 def is_mailbox(self, fn): return False def _mailbox_name(self, path): return _("Inbox") def _create_tag(self, *args, **kwargs): ptag = kwargs.get('parent') try: if ptag: return self.session.config.get_tags(ptag)[0]._key except (IndexError, KeyError): pass return BaseMailSource._create_tag(self, *args, **kwargs) def TestPop3Settings(session, settings, event): password = IndirectPassword(session.config, settings['password']) conn = _open_pop3_mailbox(session, event, settings['host'], int(settings['port']), settings['username'], password, settings.get('auth_type', 'password'), settings['protocol'], True) if conn: conn.close() return True return False ================================================ FILE: mailpile/mailboxes/__init__.py ================================================ ## Dear hackers! ## ## It would be great to have more mailbox classes. They should be derived ## from or implement the same interfaces as Python's native mailboxes, with ## the additional constraint that they support pickling and unpickling using ## cPickle. The mailbox class is also responsible for generating and parsing ## a "pointer" which should be a short as possible while still encoding the ## info required to locate this message and this message only within the ## larger mailbox. import time from urllib import quote, unquote from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.index.mailboxes import MailboxIndex from mailpile.mailutils import MBX_ID_LEN from mailpile.util import MboxRLock __all__ = ['mbox', 'maildir', 'gmvault', 'macmail', 'pop3', 'wervd', 'MBX_ID_LEN', 'NoSuchMailboxError', 'IsMailbox', 'OpenMailbox'] MAILBOX_CLASSES = [] class NoSuchMailboxError(OSError): pass def register(prio, cls): global MAILBOX_CLASSES MAILBOX_CLASSES.append((prio, cls)) MAILBOX_CLASSES.sort() def IsMailbox(fn, config): for pri, mbox_cls in MAILBOX_CLASSES: try: if mbox_cls.parse_path(config, fn): return (True, mbox_cls) except KeyboardInterrupt: raise except: pass return False def OpenMailbox(fn, config, create=False): for pri, mbox_cls in MAILBOX_CLASSES: try: return mbox_cls( *mbox_cls.parse_path(config, fn, create=create, allow_empty=True)) except KeyboardInterrupt: raise except: pass raise ValueError('Not a mailbox: %s' % fn) def UnorderedPicklable(parent, editable=False): """A factory for generating unordered, picklable mailbox classes.""" class UnorderedPicklableMailbox(parent): UNPICKLABLE = [] def __init__(self, *args, **kwargs): parent.__init__(self, *args, **kwargs) self.editable = editable self.source_map = {} self.is_local = False self._last_updated = None self._lock = MboxRLock() self._index = None self._save_to = None self._encryption_key_func = lambda: None self._decryption_key_func = lambda: None self.__init2__(*args, **kwargs) def __init2__(self, *args, **kwargs): pass def __enter__(self, *args, **kwargs): self._lock.acquire() return self def __exit__(self, *args, **kwargs): self._lock.release() def __unicode__(self): return unicode(str(self)) def describe_msg_by_ptr(self, msg_ptr): try: return self._describe_msg_by_ptr(msg_ptr) except KeyError: return _("message not found in mailbox") def _describe_msg_by_ptr(self, msg_ptr): return unicode(msg_ptr) def __setstate__(self, data): self.__dict__.update(data) self._lock = MboxRLock() with self._lock: self._index = None self._save_to = None self._encryption_key_func = lambda: None self._decryption_key_func = lambda: None if not hasattr(self, 'source_map'): self.source_map = {} if (len(self.source_map) > 0 and not hasattr(self, 'is_local') or not self.is_local): self.is_local = True self.update_toc() def __getstate__(self): odict = self.__dict__.copy() # Pickle can't handle function objects. for dk in ['_save_to', '_index', '_last_updated', '_encryption_key_func', '_decryption_key_func', '_file', '_lock', 'parsed'] + self.UNPICKLABLE: if dk in odict: del odict[dk] return odict def save(self, session=None, to=None, pickler=None): with self._lock: if to and pickler: self._save_to = (pickler, to) if self._save_to and len(self) > 0: pickler, fn = self._save_to if session: session.ui.mark(_('Saving %s state to %s') % (self, fn)) pickler(self, fn) def add_from_source(self, source_id, metadata_kws, *args, **kwargs): with self._lock: key = self.add(*args, **kwargs) self.set_metadata_keywords(key, metadata_kws) self.source_map[source_id] = key return key def update_toc(self): self._last_updated = time.time() self._refresh() self._last_updated = time.time() def last_updated(self): return self._last_updated def get_msg_ptr(self, mboxid, toc_id): return '%s%s' % (mboxid, quote(toc_id)) def get_file(self, *args, **kwargs): with self._lock: return parent.get_file(self, *args, **kwargs) def get_file_by_ptr(self, msg_ptr): return self.get_file(unquote(msg_ptr[MBX_ID_LEN:])) def remove_by_ptr(self, msg_ptr): self._last_updated = time.time() return self.remove(unquote(msg_ptr[MBX_ID_LEN:])) def get_msg_size(self, toc_id): with self._lock: fd = self.get_file(toc_id) fd.seek(0, 2) return fd.tell() def get_bytes(self, toc_id, *args): with self._lock: return self.get_file(toc_id).read(*args) def get_string(self, *args, **kwargs): with self._lock: return parent.get_string(self, *args, **kwargs) def get_metadata_keywords(self, toc_id): # Subclasses should translate whatever internal metadata they # have into mailpile keywords describing message metadata return [] def set_metadata_keywords(self, toc_id, kws): pass def get_index(self, config, mbx_mid=None): with self._lock: if self._index is None: self._index = MailboxIndex(config, self, mbx_mid=mbx_mid) return self._index def remove(self, *args, **kwargs): with self._lock: self._last_updated = time.time() return parent.remove(self, *args, **kwargs) def _get_fd(self, *args, **kwargs): with self._lock: return parent._get_fd(self, *args, **kwargs) def _refresh(self, *args, **kwargs): with self._lock: return parent._refresh(self, *args, **kwargs) def __setitem__(self, *args, **kwargs): with self._lock: self._last_updated = time.time() return parent.__setitem__(self, *args, **kwargs) def __getitem__(self, *args, **kwargs): with self._lock: return parent.__getitem__(self, *args, **kwargs) return UnorderedPicklableMailbox ================================================ FILE: mailpile/mailboxes/gmvault.py ================================================ import mailbox import os import gzip import rfc822 import mailpile.mailboxes import mailpile.mailboxes.maildir as maildir from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n class MailpileMailbox(maildir.MailpileMailbox): """A Gmvault class that supports pickling and a few mailpile specifics.""" @classmethod def parse_path(cls, config, fn, create=False, allow_empty=False): if (os.path.isdir(fn) and os.path.isdirs(os.path.join(fn, 'db')) and os.path.isdirs(os.path.join(fn, 'chats')) and os.path.isdirs(os.path.join(fn, '.info'))): return (fn, ) raise ValueError('Not a Gmvault: %s' % fn) def __init__(self, dirname, factory=rfc822.Message, create=True): maildir.MailpileMailbox.__init__(self, dirname, factory, create) self._paths = {'db': os.path.join(self._path, 'db')} self._toc_mtimes = {'db': 0} def get_file(self, key): """Return a file-like representation or raise a KeyError.""" fname = self._lookup(key) if fname.endswith('.gz'): f = gzip.open(os.path.join(self._path, fname), 'rb') else: f = open(os.path.join(self._path, fname), 'rb') return mailbox._ProxyFile(f) def _refresh(self): """Update table of contents mapping.""" # Refresh toc self._toc = {} for path in self._paths: for dirpath, dirnames, filenames in os.walk(self._paths[path]): for filename in [f for f in filenames if f.endswith(".eml.gz") or f.endswith(".eml")]: self._toc[filename] = os.path.join(dirpath, filename) mailpile.mailboxes.register(50, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/macmail.py ================================================ import mailbox import sys import os import warnings import rfc822 import time import errno import mailpile.mailboxes from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import UnorderedPicklable class _MacMaildirPartialFile(mailbox._PartialFile): def __init__(self, fd): length = int(fd.readline().strip()) start = fd.tell() stop = start+length mailbox._PartialFile.__init__(self, fd, start=start, stop=stop) class MacMaildirMessage(mailbox.Message): def __init__(self, message=None): if hasattr(message, "read"): length = int(message.readline().strip()) message = message.read(length) mailbox.Message.__init__(self, message) class MacMaildir(mailbox.Mailbox): def __init__(self, dirname, factory=rfc822.Message, create=True): mailbox.Mailbox.__init__(self, dirname, factory, create) if not os.path.exists(self._path): if create: raise NotImplemented("Why would we support creation of " "silly mailboxes?") else: raise mailbox.NoSuchMailboxError(self._path) # What have we here? ds = os.listdir(self._path) # Okay, MacMaildirs have Info.plist files if not 'Info.plist' in ds: raise mailbox.FormatError(self._path) # Now ignore all the files and dotfiles... ds = [d for d in ds if not d.startswith('.') and os.path.isdir(os.path.join(self._path, d))] # There should be exactly one directory left, which is our "ID". if len(ds) == 1: self._id = ds[0] else: raise mailbox.FormatError(self._path) # And finally, there's a Data folder (with .emlx files in it) self._mailroot = "%s/%s/Data/" % (self._path, self._id) if not os.path.isdir(self._mailroot): raise mailbox.FormatError(self._path) self._toc = {} self._last_read = 0 def remove(self, key): """Remove the message or raise error if nonexistent.""" safe_remove(os.path.join(self._mailroot, self._lookup(key))) try: del self._toc[key] except: pass def discard(self, key): """If the message exists, remove it.""" try: self.remove(key) except KeyError: pass except OSError as e: if e.errno != errno.ENOENT: raise def __setitem__(self, key, message): """Replace a message""" raise NotImplemented("Mailpile is readonly, for now.") def iterkeys(self): self._refresh() for key in self._toc: try: self._lookup(key) except KeyError: continue yield key def has_key(self, key): self._refresh() return key in self._toc def __len__(self): self._refresh() return len(self._toc) def _refresh(self): self._toc = {} paths = [""] while not paths == []: curpath = paths.pop(0) fullpath = os.path.join(self._mailroot, curpath) try: for entry in os.listdir(fullpath): p = os.path.join(fullpath, entry) if os.path.isdir(p): paths.append(os.path.join(curpath, entry)) elif entry[-5:] == ".emlx": self._toc[entry[:-5]] = os.path.join(curpath, entry) except (OSError, IOError): pass # Ignore difficulties reading individual folders def _lookup(self, key): try: if os.path.exists(os.path.join(self._mailroot, self._toc[key])): return self._toc[key] except KeyError: pass self._refresh() try: return self._toc[key] except KeyError: raise KeyError("No message with key %s" % key) def get_message(self, key): f = open(os.path.join(self._mailroot, self._lookup(key)), 'r') msg = MacMaildirMessage(f) f.close() return msg def get_file(self, key): f = open(os.path.join(self._mailroot, self._lookup(key)), 'r') return _MacMaildirPartialFile(f) class MailpileMailbox(UnorderedPicklable(MacMaildir)): """A Mac Mail.app maildir class that supports pickling etc.""" @classmethod def parse_path(cls, config, fn, create=False, allow_empty=False): if (os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'Info.plist'))): return (fn, ) raise ValueError('Not a Mac Mail.app Maildir: %s' % fn) def __unicode__(self): return _("Mac Maildir %s") % self._mailroot def _describe_msg_by_ptr(self, msg_ptr): return _("e-mail in file %s") % self._lookup(msg_ptr[MBX_ID_LEN:]) mailpile.mailboxes.register(50, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/maildir.py ================================================ import mailbox import os import sys import mailpile.mailboxes from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import UnorderedPicklable class MailpileMailbox(UnorderedPicklable(mailbox.Maildir, editable=True)): """A Maildir class that supports pickling and a few mailpile specifics.""" supported_platform = None @classmethod def parse_path(cls, config, fn, create=False, allow_empty=False): if (((cls.supported_platform is None) or (cls.supported_platform == sys.platform[:3].lower())) and ((os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'cur'))) or (create and not os.path.exists(fn)))): return (fn, ) raise ValueError('Not a Maildir: %s' % fn) def _refresh(self): with self._lock: mailbox.Maildir._refresh(self) # Dotfiles are not mail. Ignore them. for t in [k for k in self._toc.keys() if k.startswith('.')]: del self._toc[t] def __unicode__(self): return _("Maildir at %s") % self._path def _describe_msg_by_ptr(self, msg_ptr): return _("e-mail in file %s") % self._lookup(msg_ptr[MBX_ID_LEN:]) def get_metadata_keywords(self, toc_id): subdir, name = os.path.split(self._lookup(toc_id)) if self.colon in name: flags = name.split(self.colon)[-1] if flags[:2] == '2,': return ['%s:maildir' % c for c in flags[2:]] return [] mailpile.mailboxes.register(25, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/maildirwin.py ================================================ import mailpile.mailboxes import mailpile.mailboxes.maildir as maildir from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n class MailpileMailbox(maildir.MailpileMailbox): """A Maildir class for Windows (using ! instead of : in filenames)""" supported_platform = 'win' colon = '!' mailpile.mailboxes.register(20, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/mbox.py ================================================ from __future__ import print_function import errno import mailbox import os import re import threading import time import traceback from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.index.mailboxes import MailboxIndex from mailpile.mailboxes import MBX_ID_LEN, NoSuchMailboxError from mailpile.util import * class MboxIndex(MailboxIndex): pass class MailpileMailbox(mailbox.mbox): """A mbox class that supports pickling and a few mailpile specifics.""" RE_STATUS = re.compile( '^(X-)?Status:\s*\S+', flags=re.IGNORECASE|re.MULTILINE) @classmethod def parse_path(cls, config, fn, create=False, allow_empty=False): try: firstline = open(fn, 'r').readline() if firstline.startswith('From '): return (fn, ) if (allow_empty or create) and not firstline: return (fn, ) except: if create and os.path.exists(fn): return (fn, ) raise ValueError('Not an mbox: %s' % fn) def __init__(self, *args, **kwargs): self._cs = {} mailbox.mbox.__init__(self, *args, **kwargs) self.editable = False self.is_local = False self._last_updated = 0 self._mtime = 0 self._index = None self._save_to = None self._encryption_key_func = lambda: None self._decryption_key_func = lambda: None self._lock = MboxRLock() def __enter__(self, *args, **kwargs): self._lock.acquire() self.lock() return self def __exit__(self, *args, **kwargs): self.unlock() self._lock.release() def __unicode__(self): return _("Unix mbox at %s") % self._path def describe_msg_by_ptr(self, msg_ptr): try: parts, start, length = self._parse_ptr(msg_ptr) return _("message at bytes %d..%d") % (start, start + length) except KeyError: return _("message not found in mailbox") def _get_fd(self): return open(self._path, 'rb+') def __setstate__(self, dict): self.__dict__.update(dict) self._lock = MboxRLock() self.is_local = False with self._lock: self._save_to = None self._encryption_key_func = lambda: None self._decryption_key_func = lambda: None try: if not os.path.exists(self._path): raise NoSuchMailboxError(self._path) self._file = self._get_fd() except IOError as e: if e.errno == errno.ENOENT: raise NoSuchMailboxError(self._path) elif e.errno == errno.EACCES: self._file = self._get_fd() else: raise self.update_toc() def __getstate__(self): odict = self.__dict__.copy() # Pickle can't handle function objects. for dk in ('_save_to', '_index', '_last_updated', '_encryption_key_func', '_decryption_key_func', '_file', '_lock', 'parsed'): if dk in odict: del odict[dk] return odict def last_updated(self): return self._last_updated def keys(self): self.update_toc() return mailbox.mbox.keys(self) def toc_values(self): self.update_toc() return self._toc.values() def update_toc(self): fd = self._get_fd() fd.seek(0, 2) cur_length = fd.tell() cur_mtime = os.path.getmtime(self._path) try: if (self._file_length == cur_length and self._mtime == cur_mtime): return except (NameError, AttributeError): pass with self._lock: fd.seek(0) self._next_key = 0 self._toc = {} self._cs = {} data = '' start = None len_nl = 1 while (cur_length > 0): self._last_updated = time.time() line_pos = fd.tell() line = fd.readline() if line.startswith('From '): if start is not None: len_nl = ('\r' == line[-2]) and 2 or 1 cs4k = self.get_msg_cs4k(0, 0, data[:-len_nl]) self._toc[self._next_key] = (start, line_pos - len_nl) self._cs[cs4k] = self._next_key self._cs[self._next_key] = cs4k self._next_key += 1 start = line_pos data = line elif line == '': if (start is not None) and (start != line_pos): cs4k = self.get_msg_cs4k(0, 0, data[:-len_nl]) self._toc[self._next_key] = (start, line_pos - len_nl) self._cs[cs4k] = self._next_key self._cs[self._next_key] = cs4k self._next_key += 1 break elif len(data) < (4096 + len_nl): data += line self._file = fd self._file_length = fd.tell() self._mtime = cur_mtime self.save(None) def _generate_toc(self): self.update_toc() def __setitem__(self, *args, **kwargs): with self._lock: mailbox.mbox.__setitem__(self, *args, **kwargs) def __delitem__(self, *args, **kwargs): with self._lock: mailbox.mbox.__delitem__(self, *args, **kwargs) def save(self, session=None, to=None, pickler=None): if to and pickler: self._save_to = (pickler, to) if self._save_to and len(self) > 0: with self._lock: pickler, fn = self._save_to if session: session.ui.mark(_('Saving %s state to %s') % (self, fn)) pickler(self, fn) def _locked_flush_without_tempfile(self): """Dangerous, but we need this for /var/mail/USER on many Linuxes""" with open(self._path, 'rb+') as new_file: new_toc = {} for key in sorted(self._toc.keys()): start, stop = self._toc[key] new_start = new_file.tell() while True: buf = self._file.read(min(4096, stop-self._file.tell())) if buf == '': break new_file.write(buf) new_toc[key] = (new_start, new_file.tell()) new_file.truncate() self._file.seek(0, 0) self._toc = new_toc self._pending = False self._pending_sync = False def flush(self, *args, **kwargs): with self._lock: self._last_updated = time.time() try: if kwargs.get('in_place', False): self._locked_flush_without_tempfile() else: mailbox.mbox.flush(self, *args, **kwargs) except OSError: if '_create_temporary' in traceback.format_exc(): self._locked_flush_without_tempfile() else: raise self._last_updated = time.time() def clear(self, *args, **kwargs): with self._lock: mailbox.mbox.clear(self, *args, **kwargs) def get_msg_size(self, toc_id): try: with self._lock: # Note: This is 1 byte less than the TOC measures, because # the final newline is ommitted. The From line is # included though. start, stop = self._toc[toc_id] return (stop - start) except (IndexError, KeyError, IndexError, TypeError): return 0 def get_metadata_keywords(self, toc_id): # In an mbox, all metadata is in the message headers. return [] def set_metadata_keywords(self, *args, **kwargs): pass def get_index(self, config, mbx_mid=None): with self._lock: if self._index is None: self._index = MboxIndex(config, self, mbx_mid=mbx_mid) return self._index def get_msg_cs(self, start, cs_size, max_length, chars=4, data=None): """Generate a checksum of a given length, ignoring Status headers.""" if data is None: if start is None: raise IOError('No data found (start=None)') with self._lock: fd = self._file fd.seek(start, 0) data = fd.read(min(cs_size, max_length)) if data == '': raise IOError('No data found at %s:%s' % (start, max_length)) elif len(data) >= cs_size: data = data[:cs_size] return b64w(sha1b64( re.sub(self.RE_STATUS, 'Status: ?', data))[:chars]) def get_msg_cs4k(self, start, max_length, data=None): """A 48-bit (6*8) checksum of the first 4k of message data.""" return self.get_msg_cs(start, 4096, max_length, chars=8, data=data) def get_msg_cs80b(self, start, max_length, data=None): """A 24-bit (6*4) checksum of the first 80 bytes of message data.""" return self.get_msg_cs(start, 80, max_length, data=data) def get_msg_ptr(self, mboxid, toc_id, data=None): with self._lock: msg_start = self._toc[toc_id][0] msg_size = self.get_msg_size(toc_id) if (toc_id in self._cs) and (data is None): msg_cs = self._cs[toc_id] else: msg_cs = self.get_msg_cs4k(msg_start, msg_size, data=data) return '%s%s:%s:%s' % ( mboxid, b36(msg_start), b36(msg_size), msg_cs) def _parse_ptr(self, msg_ptr): parts = msg_ptr[MBX_ID_LEN:].split(':') start = int(parts[0], 36) length = int(parts[1], 36) if len(parts) > 2: if parts[2] in self._cs: start, end = self._toc[self._cs[parts[2]]] length = end - start return parts, start, length def _verify_ptr_checksums(self, msg_ptr, start, ignored_fd): """Check whether the msg_ptr checksums match the data at [start].""" with self._lock: parts, ignored_start, length = self._parse_ptr(msg_ptr) cs80b = self.get_msg_cs80b(start, length) if len(parts) > 2: cs4k = self.get_msg_cs4k(start, length) cs = parts[2] if (cs4k != cs and cs80b != cs): return False return True def _possible_message_locations(self, msg_ptr, max_locations=15): """Yield possible locations for messages of a given size.""" with self._lock: parts, pstart, length = self._parse_ptr(msg_ptr) # This is where it is SUPPOSED to be, always check that first. starts = [pstart] # Extend the list with other messages of the right size. # We accept two lengths, because there were off-by-one errors # in older versions of Mailpile. :-( starts.extend(sorted([ b for b, e in self.toc_values() if length in (e-b, e-b+1) and b != pstart])) # Yield up to max_locations positions for i, start in enumerate(starts[:max_locations]): yield (start, length) def _get_SSLP_by_ptr(self, msg_ptr, verifier=None, from_=False): if verifier is None: verifier = self._verify_ptr_checksums tries = [] length = None for from_start, length in self._possible_message_locations(msg_ptr): # We duplicate the file descriptor here, in case other threads # are accessing the same mailbox and moving it around, or in # case we have multiple PartialFile objects in flight at once. tries.append(str(from_start)) try: start = from_start stop = from_start + length fd = self._get_fd() if not from_: fd.seek(start) length -= len(fd.readline()) start = fd.tell() pf = mailbox._PartialFile(fd, start, stop) if verifier(msg_ptr, from_start, pf): return (from_start, start, length, pf) except IOError: pass err = '%s: %s %s@%s' % ( _('Message not found'), msg_ptr, length, '/'.join(tries)) raise IOError(err) def update(self, *args, **kwargs): with self._lock: self._cs = {} # FIXME return mailbox.mbox.update(self, *args, **kwargs) def discard(self, *args, **kwargs): with self._lock: self._cs = {} # FIXME return mailbox.mbox.discard(self, *args, **kwargs) def remove(self, *args, **kwargs): with self._lock: self._cs = {} # FIXME return mailbox.mbox.remove(self, *args, **kwargs) def get_file_by_ptr(self, msg_ptr, verifier=None, from_=False): with self._lock: from_start, start, length, pfile = self._get_SSLP_by_ptr( msg_ptr, verifier=verifier, from_=from_) return pfile def remove_by_ptr(self, msg_ptr): with self._lock: from_start, start, length, pfile = self._get_SSLP_by_ptr(msg_ptr) keys = [k for k in self._toc if self._toc[k][0] == from_start] if keys: return self.remove(keys[0]) raise KeyError('Not found: %s' % msg_ptr) def get_bytes(self, toc_id, *args, **kwargs): with self._lock: return self.get_file(toc_id, *args, **kwargs).read() def get_file(self, *args, **kwargs): with self._lock: return mailbox.mbox.get_file(self, *args, **kwargs) if __name__ == "__main__": import tempfile, time, sys verbose = ('-v' in sys.argv) or ('--verbose' in sys.argv) wait = ('-w' in sys.argv) or ('--wait' in sys.argv) MSG_TEMPLATE = """\ From bre@mailpile.is Mon Jan 1 08:14:00 2018 Return-Path: Subject: %(subject)s Message-ID: <%(msgid)s> Content-Length: %(length)s %(content)s""" problems = tests = 0 with tempfile.NamedTemporaryFile() as tf: lengths = [] for count in range(0, 35): body = ''.join([ 'Hello world, this is a message!\n' ] * ((27 * (100-count)) % 1230)) message = (MSG_TEMPLATE % { 'subject': 'Test message #%d' % count, 'msgid': '%d@example.com' % count, 'length': len(body), 'content': body}) lengths.append(len(message)) tf.write(message) tf.write("\n") tf.flush() if verbose or wait: print('Temporary mailbox in: %s' % tf.name) if wait: raw_input('Press ENTER to continue...') pmbx = mailbox.mbox(tf.name) mmbx = MailpileMailbox(tf.name) ptrs = [] for i, key in enumerate(mmbx.keys()): msg_ptr = mmbx.get_msg_ptr('0000', key) o_size = lengths[i] c_size = mmbx.get_msg_size(key) f_size = len(mmbx.get_bytes(key, from_=True)) f2size = len(mmbx.get_file_by_ptr(msg_ptr, from_=True).read()) result = 'ok' if (o_size == c_size == f_size == f2size) else 'BAD' if verbose or result != 'ok': print("%-3.3s [%s/%s/%s] %s ?= %s ?= %s ?= %s" % ( result, i, key, msg_ptr, o_size, c_size, f_size, f2size)) if result != 'ok': problems += 1 tests += 1 ptrs.append([msg_ptr, f2size]) # Remove some messages, bypassing MailpileMailbox deletions = [0, 5, 10, 15, 34] for d in reversed(sorted(deletions)): del pmbx[d] pmbx.flush() # Remove a message using MailpileMailbox try: tests += 1 deletions.append(1) mmbx.remove_by_ptr(ptrs[1][0]) mmbx.flush() except KeyError: problems += 1 for i, (msg_ptr, f2size) in enumerate(ptrs): tests += 1 if i in deletions: try: mmbx.get_file_by_ptr(msg_ptr, from_=True).read() problems += 1 print('BAD Found deleted message %s' % msg_ptr) except IOError: if verbose: print('ok IOError on message %s' % msg_ptr) continue f3size = len(mmbx.get_file_by_ptr(msg_ptr, from_=True).read()) if (f2size != f3size): problems += 1 print('BAD Message %s: wrong size in new location' % msg_ptr) elif verbose: print('ok Message %s found in new location' % msg_ptr) # This is formatted to look like doctest results... print('TestResults(failed=%d, attempted=%d)' % (problems, tests)) if wait: raw_input('Tests finished. Press ENTER to clean up...') if problems: sys.exit(1) else: import mailpile.mailboxes mailpile.mailboxes.register(90, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/pop3.py ================================================ from __future__ import print_function try: import cStringIO as StringIO except ImportError: import StringIO import poplib import socket import ssl import time from mailbox import Mailbox, Message import mailpile.mailboxes from mailpile.conn_brokers import Master as ConnBroker from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import UnorderedPicklable from mailpile.util import * class wrappable_POP3_SSL(poplib.POP3_SSL): """ Override the default poplib.POP3_SSL init to use socket.create_connection """ def __init__(self, host, port=poplib.POP3_SSL_PORT, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): self.host = host self.port = port self.keyfile = keyfile self.certfile = certfile self.buffer = "" self.sock = socket.create_connection((host, port), timeout) self.file = self.sock.makefile('rb') self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile) self._debugging = 0 self.welcome = self._getresp() class UnsupportedProtocolError(Exception): pass class POP3Mailbox(Mailbox): """ Basic implementation of POP3 Mailbox. """ def __init__(self, host, user=None, password=None, auth_type='password', use_ssl=True, port=None, debug=False, conn_cls=None, session=None): """Initialize a Mailbox instance.""" Mailbox.__init__(self, '/') self.host = host self.user = user self.password = password self.auth_type = auth_type self.use_ssl = use_ssl self.port = port self.debug = debug self.conn_cls = conn_cls self.session = session self._lock = MboxRLock() self._pop3 = None self._connect() def lock(self): pass def unlock(self): pass def _connect(self): with self._lock: if self._pop3: try: self._pop3.noop() return except poplib.error_proto: self._pop3 = None with ConnBroker.context(need=[ConnBroker.OUTGOING_POP3]): if self.conn_cls: self._pop3 = self.conn_cls(self.host, self.port or 110, timeout=120) self.secure = self.use_ssl elif self.use_ssl: self._pop3 = wrappable_POP3_SSL(self.host, self.port or 995, timeout=120) self.secure = True else: self._pop3 = poplib.POP3(self.host, self.port or 110, timeout=120) self.secure = False if hasattr(self._pop3, 'sock'): self._pop3.sock.settimeout(120) if self.debug: self._pop3.set_debuglevel(self.debug) self._keys = None try: if self.auth_type.lower() == 'oauth2': from mailpile.plugins.oauth import OAuth2 token_info = OAuth2.GetFreshTokenInfo(self.session, self.user) if self.user and token_info and token_info.access_token: raise AccessError("FIXME: Do OAUTH2 Auth!") else: raise AccessError() else: self._pop3.user(self.user) self._pop3.pass_(self.password.encode('utf-8')) except poplib.error_proto: raise AccessError() def _refresh(self): with self._lock: self._keys = None self.iterkeys() def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" raise NotImplementedError('Method must be implemented by subclass') def _get(self, key, _bytes=None): with self._lock: if key not in self.iterkeys(): raise KeyError('Invalid key: %s' % key) self._connect() if _bytes is not None: lines = max(10, _bytes//30) # A wild guess! ok, lines, octets = self._pop3.top(self._km[key], lines) else: ok, lines, octets = self._pop3.retr(self._km[key]) if not ok.startswith('+OK'): raise KeyError('Invalid key: %s' % key) # poplib is stupid in that it loses the linefeeds, so we need to # do some guesswork to bring them back to what the server provided. # If we don't do this jiggering, then sizes don't match up, which # could cause allocation bugs down the line. have_octets = sum(len(l) for l in lines) if octets == have_octets + len(lines): lines.append('') data = '\n'.join(lines) elif octets == have_octets + 2*len(lines): lines.append('') data = '\r\n'.join(lines) elif octets == have_octets + len(lines) - 1: data = '\n'.join(lines) elif octets == have_octets + 2*len(lines) - 2: data = '\r\n'.join(lines) else: raise ValueError('Length mismatch in message %s' % key) if _bytes is not None: return data[:_bytes] else: return data def get_message(self, key): """Return a Message representation or raise a KeyError.""" return Message(self._get(key)) def get_bytes(self, key, *args): """Return a byte string representation or raise a KeyError.""" return self._get(key, *args) def get_file(self, key): """Return a file-like representation or raise a KeyError.""" return StringIO.StringIO(self._get(key)) def get_msg_size(self, key): with self._lock: self._connect() if key not in self.iterkeys(): raise KeyError('Invalid key: %s' % key) ok, info, octets = self._pop3.list(self._km[key]).split() return int(octets) def remove(self, key): # FIXME: This is very inefficient if we are deleting multiple # messages at once. with self._lock: self._connect() if key not in self.iterkeys(): raise KeyError('Invalid key: %s' % key) ok = self._pop3.dele(self._km[key]) self._refresh() def stat(self): with self._lock: self._connect() return self._pop3.stat() def iterkeys(self): """Return an iterator over keys.""" # Note: POP3 *without UIDL* is useless. We don't support it. with self._lock: if self._keys is None: self._connect() try: stat, key_list, octets = self._pop3.uidl() except poplib.error_proto: raise UnsupportedProtocolError() self._keys = [tuple(k.split(' ', 1)) for k in key_list] self._km = dict([reversed(k) for k in self._keys]) return [k[1] for k in self._keys] def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" return key in self.iterkeys() def __len__(self): """Return a count of messages in the mailbox.""" return len(self.iterkeys()) def flush(self): """Write any pending changes to the disk.""" self.close() def close(self): """Flush and close the mailbox.""" try: if self._pop3: self._pop3.quit() finally: self._pop3 = None self._keys = None class MailpileMailbox(UnorderedPicklable(POP3Mailbox)): UNPICKLABLE = ['_pop3', '_debug'] @classmethod def parse_path(cls, config, path, create=False, allow_empty=False): path = path.split('/') if path and path[0].lower() in ('pop:', 'pop3:', 'pop3_ssl:', 'pop3s:'): proto = path[0][:-1].lower() userpart, server = path[2].rsplit("@", 1) user, password = userpart.rsplit(":", 1) if ":" in server: server, port = server.split(":", 1) else: port = 995 if ('s' in proto) else 110 # This is a hack for GMail if 'recent' in path[3:]: user = 'recent:' + user if not config: debug = False elif 'pop3' in config.sys.debug: debug = 99 elif 'rescan' in config.sys.debug: debug = 1 else: debug = False # WARNING: Order must match POP3Mailbox.__init__(...) return (server, user, password, 's' in proto, int(port), debug) raise ValueError('Not a POP3 url: %s' % path) def save(self, *args, **kwargs): # Do not save state locally pass ##[ Test code follows ]####################################################### if __name__ == "__main__": import doctest import sys class _MockPOP3(object): """ Base mock that pretends to be a poplib POP3 connection. >>> pm = POP3Mailbox('localhost', user='bad', conn_cls=_MockPOP3) Traceback (most recent call last): ... AccessError >>> pm = POP3Mailbox('localhost', user='a', password='b', ... conn_cls=_MockPOP3) >>> pm.stat() (2, 123456) >>> pm.iterkeys() ['evil', 'good'] >>> 'evil' in pm, 'bogon' in pm (True, False) >>> [msg['subject'] for msg in pm] ['Msg 1', 'Msg 2'] >>> pm.get_msg_size('evil'), pm.get_msg_size('good') (47, 51) >>> pm.get_bytes('evil') 'From: test@mailpile.is\\nSubject: Msg 1\\n\\nOh, hi!\\n' >>> pm.get_bytes('evil', 5) 'From:' >>> pm['invalid-key'] Traceback (most recent call last): ... KeyError: ... """ TEST_MSG = ('From: test@mailpile.is\r\n' 'Subject: Msg N\r\n' '\r\n' 'Oh, hi!\r\n') DEFAULT_RESULTS = { 'user': lambda s, u: '+OK' if (u == 'a') else '-ERR', 'pass_': lambda s, u: '+OK Logged in.' if (u == 'b') else '-ERR', 'stat': (2, 123456), 'noop': '+OK', 'list_': lambda s: ('+OK 2 messages:', ['1 %d' % len(s.TEST_MSG.replace('\r', '')), '2 %d' % len(s.TEST_MSG)], 0), 'uidl': ('+OK', ['1 evil', '2 good'], 0), 'retr': lambda s, m: ('+OK', s.TEST_MSG.replace('N', m).splitlines(), len(s.TEST_MSG) if m[0] == '2' else len(s.TEST_MSG.replace('\r', ''))), 'top': lambda s, m, n: ('+OK', s.TEST_MSG.splitlines()[:n], len(''.join(s.TEST_MSG.splitlines(1)[:n]))), } RESULTS = {} def __init__(self, *args, **kwargs): def mkcmd(rval): def r(rv): if isinstance(rv, (str, unicode)) and rv[0] != '+': raise poplib.error_proto(rv) return rv def cmd(*args, **kwargs): if isinstance(rval, (str, unicode, list, tuple, dict)): return r(rval) else: return r(rval(self, *args, **kwargs)) return cmd for cmd, rval in dict_merge(self.DEFAULT_RESULTS, self.RESULTS ).iteritems(): self.__setattr__(cmd, mkcmd(rval)) def list(self, which=None): msgs = self.list_() if which: return '+OK ' + msgs[1][1-int(which)] return msgs def __getattr__(self, attr): return self.__getattribute__(attr) class _MockPOP3_Without_UIDL(_MockPOP3): """ Mock that lacks the UIDL command. >>> pm = POP3Mailbox('localhost', user='a', password='b', ... conn_cls=_MockPOP3_Without_UIDL) >>> pm.iterkeys() Traceback (most recent call last): ... UnsupportedProtocolError """ RESULTS = {'uidl': '-ERR'} results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) if len(sys.argv) > 1: mbx = MailpileMailbox(*MailpileMailbox.parse_path(None, sys.argv[1])) print('Status is: %s' % (mbx.stat(), )) print('Downloading mail and listing subjects, hit CTRL-C to quit') for msg in mbx: print(msg['subject']) time.sleep(2) else: mailpile.mailboxes.register(10, MailpileMailbox) ================================================ FILE: mailpile/mailboxes/wervd.py ================================================ import email.generator import email.message import mailbox import StringIO import sys import mailpile.mailboxes from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import UnorderedPicklable, MBX_ID_LEN from mailpile.crypto.streamer import * from mailpile.util import safe_remove class MailpileMailbox(UnorderedPicklable(mailbox.Maildir, editable=True)): """A Maildir class that supports pickling and a few mailpile specifics.""" supported_platform = None colon = '!' # Works on both Windows and Unix # FIXME: Copies were part of the original WERVD spec, to compensate for # the additional fragility of encrypted data. This hasn't been # implemented however, and while SSDs are expensive it is not # obvious that doubling (or tripling...) the storage requirements # for all e-mail is a cost folks are willing to pay for never # losing a message to the occaisional bitflips. So this is all # commented out at the moment. Revisit? # MAX_COPIES = 5 @classmethod def parse_path(cls, config, fn, create=False, allow_empty=False): if (((cls.supported_platform is None) or (cls.supported_platform == sys.platform[:3].lower())) and ((os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'cur')) and os.path.exists(os.path.join(fn, 'wervd.ver'))) or (create and not os.path.exists(fn)))): return (fn, ) raise ValueError('Not a Mailpile Maildir: %s' % fn) def __init2__(self, *args, **kwargs): open(os.path.join(self._path, 'wervd.ver'), 'w+b').write('0') def __unicode__(self): return _("Mailpile mailbox at %s") % self._path def _describe_msg_by_ptr(self, msg_ptr): return _("e-mail in file %s") % self._lookup(msg_ptr[MBX_ID_LEN:]) # FIXME: Copies # def _copy_paths(self, where, key, copies): # for cpn in range(1, copies): # yield os.path.join(self._path, where, '%s.%s' % (key, cpn)) def remove(self, key): with self._lock: fn = os.path.join(self._path, self._lookup(key)) del self._toc[key] safe_remove(fn) # FIXME: Copies # # Also remove all the copies of this message! # key = os.path.basename(fn) # for where in ('cur', 'new', 'tmp'): # for copy_fn in self._copy_paths(where, key, self.MAX_COPIES): # if os.path.exists(copy_fn): # safe_remove(copy_fn) # else: # break def _refresh(self): with self._lock: mailbox.Maildir._refresh(self) # WERVD mail names don't have dots in them for t in [k for k in self._toc.keys() if '.' in k]: del self._toc[t] safe_remove() # Try to remove any postponed removals def _get_fd(self, key): with self._lock: fn = os.path.join(self._path, self._lookup(key)) mep_key = self._decryption_key_func() fd = open(fn, 'rb') if mep_key: fd = DecryptingStreamer(fd, mep_key=mep_key, name='WERVD(%s)' % fn) return fd def get_message(self, key): """Return a Message representation or raise a KeyError.""" with self._lock: with self._get_fd(key) as fd: if self._factory: return self._factory(fd) else: return mailbox.MaildirMessage(fd) def get_string(self, key): with self._lock: with self._get_fd(key) as fd: return fd.read() def get_file(self, key): with self._lock: return StringIO.StringIO(self.get_string(key)) def get_metadata_keywords(self, toc_id): subdir, name = os.path.split(self._lookup(toc_id)) if self.colon in name: flags = name.split(self.colon)[-1] if flags[:2] == '2,': return ['%s:maildir' % c for c in flags[2:]] return [] def set_metadata_keywords(self, toc_id, kws): with self._lock: old_fpath = self._lookup(toc_id) new_fpath = old_fpath.rsplit(self.colon, 1)[0] flags = ''.join(sorted([k[0] for k in kws])) if flags: new_fpath += '%s2,%s' % (self.colon, flags) if new_fpath != old_fpath: os.rename(os.path.join(self._path, old_fpath), os.path.join(self._path, new_fpath)) self._toc[toc_id] = new_fpath def add(self, message): """Add message and return assigned key.""" key = self._encryption_key_func() es = None try: tmpdir = os.path.join(self._path, 'tmp') if not os.path.exists(tmpdir): os.mkdir(tmpdir, 0o700) if key: es = EncryptingStreamer(key, dir=tmpdir, name='WERVD', delimited=False) else: es = ChecksummingStreamer(dir=tmpdir, name='WERVD') self._dump_message(message, es) es.finish() # We are using the MAC to detect file system corruption, not in a # security context - so using as little as 40 bits should be fine. saved = False key = None outer_mac = es.outer_mac_sha256() for l in range(10, len(outer_mac)): key = outer_mac[:l] fn = os.path.join(self._path, 'new', key) if not os.path.exists(fn): es.save(fn) saved = self._toc[key] = os.path.join('new', key) break if not saved: raise mailbox.ExternalClashError(_('Could not find a filename ' 'for the message.')) # FIXME: Copies # for fn in self._copy_paths('new', key, copies): # with mailbox._create_carefully(fn) as ofd: # es.save_copy(ofd) return key finally: if es is not None: es.close() def _dump_message(self, message, target): if isinstance(message, email.message.Message): gen = email.generator.Generator(target, False, 0) gen.flatten(message) elif isinstance(message, str): target.write(message) else: raise TypeError(_('Invalid message type: %s') % type(message)) def __setitem__(self, key, message): raise IOError(_('Mailbox messages are immutable')) mailpile.mailboxes.register(15, MailpileMailbox) ================================================ FILE: mailpile/mailutils/__init__.py ================================================ # vim: set fileencoding=utf-8 : # MBX_ID_LEN = 4 # 4x36 == 1.6 million mailboxes def FormatMbxId(n): if not isinstance(n, (str, unicode)): n = b36(n) if len(n) > MBX_ID_LEN: raise ValueError(_('%s is too large to be a mailbox ID') % n) return ('0000' + n).lower()[-MBX_ID_LEN:] class NotEditableError(ValueError): pass class NoFromAddressError(ValueError): pass class NoRecipientError(ValueError): pass class InsecureSmtpError(IOError): def __init__(self, msg, details=None): IOError.__init__(self, msg) self.error_info = details or {} class NoSuchMailboxError(OSError): pass ================================================ FILE: mailpile/mailutils/addresses.py ================================================ # vim: set fileencoding=utf-8 : # from __future__ import print_function import base64 import copy import quopri import re from mailpile.util import * from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.vcard import AddressInfo class AddressHeaderParser(list): """ This is a class which tries very hard to interpret the From:, To: and Cc: lines found in real-world e-mail and make sense of them. The general strategy of this parser is to: 1. parse header data into tokens 2. group tokens together into address + name constructs. And optionaly, 3. normalize each group to a standard format In practice, we do this in multiple passes: first a strict pass where we try to parse things semi-sensibly, followed by fuzzier heuristics. Ideally, if folks format things correctly we should parse correctly. But if that fails, there are are other passes where we try to cope with various types of weirdness we've seen in the wild. The wild can be pretty wild. This parser is NOT (yet) fully RFC2822 compliant - in particular it will get confused by nested comments (see FIXME in tests below). The normalization will take pains to ensure that < and , are never present inside a name/comment (even if legal), to make life easier for lame parsers down the line. Examples: >>> ahp = AddressHeaderParser(AddressHeaderParser.TEST_HEADER_DATA) >>> ai = ahp[1] >>> ai.fn u'Bjarni' >>> ai.address u'bre@klaki.net' >>> ahpn = ahp.normalized_addresses() >>> (ahpn == ahp.TEST_EXPECT_NORMALIZED_ADDRESSES) or ahpn True >>> AddressHeaderParser('Weird email@somewhere.com Header').normalized() u'"Weird Header" ' >>> ai = AddressHeaderParser(unicode_data=ahp.TEST_UNICODE_DATA) >>> ai[0].fn u'Bjarni R\\xfanar' >>> ai[0].fn == ahp.TEST_UNICODE_NAME True >>> ai[0].address u'b@c.x' """ TEST_UNICODE_DATA = u'Bjarni R\xfanar ' TEST_UNICODE_NAME = u'Bjarni R\xfanar' TEST_HEADER_DATA = """ bre@klaki.net , bre@klaki.net Bjarni , bre@klaki.net bre@klaki.net, bre@klaki.net (bre@notmail.com), "" , =?utf-8?Q?=3Cbre@notmail.com=3E?= , bre@klaki.net ((nested) bre@notmail.com comment), (FIXME: (nested) bre@wrongmail.com parser breaker) bre@klaki.net, undisclosed-recipients-gets-ignored:, Bjarni [mailto:bre@klaki.net], "This is a key test" , bre@klaki.net (Bjarni Runar Einar's son); Bjarni =?iso-8859-1?Q??=is bre @klaki.net, Bjarni =?iso-8859-1?Q?Runar?=Einarsson<' bre'@ klaki.net>, "Einarsson, Bjarni" , =?iso-8859-1?Q?Lonia_l=F6gmannsstofa?= , "Bjarni @ work" , """ TEST_EXPECT_NORMALIZED_ADDRESSES = [ '', '"Bjarni" ', '"bre@klaki.net" ', '"bre@notmail.com" ', '=?utf-8?Q?=3Cbre@notmail.com=3E?= ', '=?utf-8?Q?=3Cbre@notmail.com=3E?= ', '"(nested bre@notmail.com comment)" ', '"(FIXME: nested parser breaker) bre@klaki.net" ', '"Bjarni" ', '"This is a key test" ', '"Bjarni Runar Einar\\\'s son" ', '"Bjarni is" ', '"Bjarni Runar Einarsson" ', '=?utf-8?Q?Einarsson=2C_Bjarni?= ', '=?utf-8?Q?Lonia_l=C3=B6gmannsstofa?= ', '"Bjarni @ work" '] # Escaping and quoting TXT_RE_QUOTE = '=\\?([^\\?\\s]+)\\?([QqBb])\\?([^\\?\\s]*)\\?=' TXT_RE_QUOTE_NG = TXT_RE_QUOTE.replace('(', '(?:') RE_ESCAPES = re.compile('\\\\([\\\\"\'])') RE_QUOTED = re.compile(TXT_RE_QUOTE) RE_SHOULD_ESCAPE = re.compile('([\\\\"\'])') RE_SHOULD_QUOTE = re.compile('[^a-zA-Z0-9()\.:/_ \'"+@-]') # This is how we normally break a header line into tokens RE_TOKENIZER = re.compile('(<[^<>]*>' # '|\\([^\\(\\)]*\\)' # (stuff) '|\\[[^\\[\\]]*\\]' # [stuff] '|"(?:\\\\\\\\|\\\\"|[^"])*"' # "stuff" "|'(?:\\\\\\\\|\\\\'|[^'])*'" # 'stuff' '|' + TXT_RE_QUOTE_NG + # =?stuff?= '|,' # , '|;' # ; '|\\s+' # white space '|[^\\s;,]+' # non-white space ')') # Where to insert spaces to help the tokenizer parse bad data RE_MUNGE_TOKENSPACERS = (re.compile('(\S)(<)'), re.compile('(\S)(=\\?)')) # Characters to strip aware entirely when tokenizing munged data RE_MUNGE_TOKENSTRIPPERS = (re.compile('[<>"]'),) # This is stuff we ignore (undisclosed-recipients, etc) RE_IGNORED_GROUP_TOKENS = re.compile('(?i)undisclosed') # Things we strip out to try and un-mangle e-mail addresses when # working with bad data. RE_MUNGE_STRIP = re.compile('(?i)(?:\\bmailto:|[\\s"\']|\?$)') # This a simple regular expression for detecting e-mail addresses. RE_MAYBE_EMAIL = re.compile('^[^()<>@,;:\\\\"\\[\\]\\s\000-\031]+' '@[a-zA-Z0-9_\\.-]+(?:#[A-Za-z0-9]+)?$') # We try and interpret non-ascii data as a particular charset, in # this order by default. Should be overridden whenever we have more # useful info from the message itself. DEFAULT_CHARSET_ORDER = ('iso-8859-1', 'utf-8') def __init__(self, data=None, unicode_data=None, charset_order=None, **kwargs): self.charset_order = charset_order or self.DEFAULT_CHARSET_ORDER self._parse_args = kwargs if data is None and unicode_data is None: self._reset(**kwargs) elif data is not None: self.parse(data) else: self.charset_order = ['utf-8'] self.parse(unicode_data.encode('utf-8')) def _reset(self, _raw_data=None, strict=False, _raise=False): self._raw_data = _raw_data self._tokens = [] self._groups = [] self[:] = [] def parse(self, data): return self._parse(data, **self._parse_args) def _parse(self, data, strict=False, _raise=False): self._reset(_raw_data=data) # 1st pass, strict try: self._tokens = self._tokenize(self._raw_data) self._groups = self._group(self._tokens) self[:] = self._find_addresses(self._groups, _raise=(not strict)) return self except ValueError: if strict and _raise: raise if strict: return self # 2nd & 3rd passes; various types of sloppy for _pass in ('2', '3'): try: self._tokens = self._tokenize(self._raw_data, munge=_pass) self._groups = self._group(self._tokens, munge=_pass) self[:] = self._find_addresses(self._groups, munge=_pass, _raise=_raise) return self except ValueError: if _pass == '3' and _raise: raise return self def unquote(self, string, charset_order=None): def uq(m): cs, how, data = m.group(1), m.group(2), m.group(3) if how in ('b', 'B'): try: data = base64.b64decode(''.join(data.split())+'===') except TypeError: print('FAILED TO B64DECODE: %s' % data) else: data = quopri.decodestring(data, header=True) try: return data.decode(cs) except LookupError: return data.decode('iso-8859-1') # Always works for cs in charset_order or self.charset_order: try: string = string.decode(cs) break except UnicodeDecodeError: pass return re.sub(self.RE_QUOTED, uq, string) @classmethod def unescape(self, string): return re.sub(self.RE_ESCAPES, lambda m: m.group(1), string) @classmethod def escape(self, strng): return re.sub(self.RE_SHOULD_ESCAPE, lambda m: '\\'+m.group(0), strng) @classmethod def quote(self, strng): if re.search(self.RE_SHOULD_QUOTE, strng): enc = quopri.encodestring(strng.encode('utf-8'), False, header=True) enc = enc.replace('<', '=3C').replace('>', '=3E') enc = enc.replace(',', '=2C') return '=?utf-8?Q?%s?=' % enc else: return '"%s"' % self.escape(strng) def _tokenize(self, string, munge=False): if munge: for ts in self.RE_MUNGE_TOKENSPACERS: string = re.sub(ts, '\\1 \\2', string) if munge == 3: for ts in self.RE_MUNGE_TOKENSTRIPPERS: string = re.sub(ts, '', string) return re.findall(self.RE_TOKENIZER, string) def _clean(self, token): if token[:1] in ('"', "'"): if token[:1] == token[-1:]: return self.unescape(token[1:-1]) elif token.startswith('[mailto:') and token[-1:] == ']': # Just convert [mailto:...] crap into a
return '<%s>' % token[8:-1] elif (token[:1] == '[' and token[-1:] == ']'): return token[1:-1] return token def _group(self, tokens, munge=False): groups = [[]] for token in tokens: token = token.strip() if token in (',', ';'): # Those tokens SHOULD separate groups, but we don't like to # create groups that have no e-mail addresses at all. if groups[-1]: if [g for g in groups[-1] if '@' in g]: groups.append([]) continue # However, this stuff is just begging to be ignored. elif [g for g in groups[-1] if re.match(self.RE_IGNORED_GROUP_TOKENS, g)]: groups[-1] = [] continue if token: groups[-1].append(self.unquote(self._clean(token))) if not groups[-1]: groups.pop(-1) return groups def _find_addresses(self, groups, **fa_kwargs): alist = [self._find_address(g, **fa_kwargs) for g in groups] return [a for a in alist if a] def _find_address(self, g, _raise=False, munge=False): if g: g = g[:] else: return [] def email_at(i): for j in range(0, len(g)): if g[j][:1] == '(' and g[j][-1:] == ')': g[j] = g[j][1:-1] rest = ' '.join([g[j] for j in range(0, len(g)) if (j != i) and g[j] ]).replace(' ,', ',').replace(' ;', ';') email, keys = g[i], None if '#' in email[email.index('@'):]: email, key = email.rsplit('#', 1) keys = [{'fingerprint': key}] return AddressInfo(email, rest.strip(), keys=keys) def munger(string): if munge: return re.sub(self.RE_MUNGE_STRIP, '', string) else: return string # If munging, look for email @domain.com in two parts, rejoin if munge: for i in range(0, len(g)): if i > 0 and i < len(g) and g[i][:1] == '@': g[i-1:i+1] = [g[i-1]+g[i]] elif i < len(g)-1 and g[i][-1:] == '@': g[i:i+2] = [g[i]+g[i+1]] # 1st, look for # # We search from the end, to make the algorithm stable in the case # that the name part also starts with a < (is that allowed?). # for i in reversed(range(0, len(g))): if g[i][:1] == '<' and g[i][-1:] == '>': maybemail = munger(g[i][1:-1]) if re.match(self.RE_MAYBE_EMAIL, maybemail): g[i] = maybemail return email_at(i) # 2nd, look for bare email@domain.com for i in range(0, len(g)): maybemail = munger(g[i]) if re.match(self.RE_MAYBE_EMAIL, maybemail): g[i] = maybemail return email_at(i) if _raise: raise ValueError('No email found in %s' % (g,)) else: return None def addresses_list(self, with_keys=False): addresses = [] for addr in self: m = addr.address if with_keys and addr.keys: m += "#" + addr.keys[0].get('fingerprint') addresses.append(m) return addresses def normalized_addresses(self, addresses=None, quote=True, with_keys=False, force_name=False): if addresses is None: addresses = self elif not addresses: addresses = [] def fmt(ai): email = ai.address if with_keys and ai.keys: fp = ai.keys[0].get('fingerprint') epart = '<%s%s>' % (email, fp and ('#%s' % fp) or '') else: epart = '<%s>' % email if ai.fn: return ' '.join([quote and self.quote(ai.fn) or ai.fn, epart]) elif force_name: return ' '.join([quote and self.quote(email) or email, epart]) else: return epart return [fmt(ai) for ai in addresses] def normalized(self, **kwargs): return ', '.join(self.normalized_addresses(**kwargs)) if __name__ == "__main__": import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/emails.py ================================================ # vim: set fileencoding=utf-8 : # # FIXME: Refactor this monster into mailpile.mailutils.* # from __future__ import print_function import base64 import copy import email.header import email.parser import email.utils import errno import mailbox import mimetypes import os import quopri import random import re import StringIO import threading import traceback from email import encoders from email.mime.base import MIMEBase from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.application import MIMEApplication from mailpile.util import * from platform import system from urllib import quote, unquote from datetime import datetime, timedelta from mailpile.crypto.gpgi import GnuPG from mailpile.crypto.mime import UnwrapMimeCrypto, MessageAsString from mailpile.crypto.state import EncryptionInfo, SignatureInfo from mailpile.eventlog import GetThreadEvent from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.vcard import AddressInfo from mailpile.mailutils import * from mailpile.mailutils.addresses import AddressHeaderParser from mailpile.mailutils.generator import Generator from mailpile.mailutils.html import extract_text_from_html, clean_html from mailpile.mailutils.headerprint import HeaderPrints from mailpile.mailutils.safe import safe_decode_hdr from mailpile.mailutils.vcal import calendar_parse GLOBAL_CONTENT_ID_LOCK = MboxLock() GLOBAL_CONTENT_ID = random.randint(0, 0xfffffff) def MakeContentID(): global GLOBAL_CONTENT_ID with GLOBAL_CONTENT_ID_LOCK: GLOBAL_CONTENT_ID += 1 GLOBAL_CONTENT_ID %= 0xfffffff return '%x' % GLOBAL_CONTENT_ID def MakeBoundary(): return '==%s==' % okay_random(30) def MakeMessageID(): # We generate a message-ID which is almost entirely random; we # include an element of the local time (give-or-take 36 hours) # to further reduce the odds of any collision. return '<%s%x@mailpile>' % ( okay_random(40), time.time() // (3600*48)) def MakeMessageDate(ts=None): # Generate valid dates, but add some jitter to the seconds field # so we're not trivially leaking our exact time. We also avoid # leaking the time zone. return email.utils.formatdate( timeval=(ts or time.time()) + (random.randint(0, 60) - 30), localtime=False) GLOBAL_PARSE_CACHE_LOCK = MboxLock() GLOBAL_PARSE_CACHE = [] def ClearParseCache(cache_id=None, pgpmime=False, full=False): global GLOBAL_PARSE_CACHE with GLOBAL_PARSE_CACHE_LOCK: GPC = GLOBAL_PARSE_CACHE for i in range(0, len(GPC)): if (full or (pgpmime and GPC[i][1]) or (cache_id and GPC[i][0] == cache_id)): GPC[i] = (None, None, None) def ParseMessage(fd, cache_id=None, update_cache=False, pgpmime='all', config=None, event=None, allow_weak_crypto=False): global GLOBAL_PARSE_CACHE if not GnuPG: pgpmime = False if cache_id is not None and not update_cache: with GLOBAL_PARSE_CACHE_LOCK: for cid, pm, message in GLOBAL_PARSE_CACHE: if cid == cache_id and pm == pgpmime: return message if pgpmime: message = ParseMessage(fd, cache_id=cache_id, pgpmime=False, config=config) if message is None: return None if cache_id is not None: # Caching is enabled, let's not clobber the encrypted version # of this message with a fancy decrypted one. message = copy.deepcopy(message) def MakeGnuPG(*args, **kwargs): ev = event or GetThreadEvent() if ev and 'event' not in kwargs: kwargs['event'] = ev return GnuPG(config, *args, **kwargs) unwrap_attachments = ('all' in pgpmime or 'att' in pgpmime) UnwrapMimeCrypto(message, protocols={'openpgp': MakeGnuPG}, unwrap_attachments=unwrap_attachments, require_MDC=(not allow_weak_crypto)) else: try: if not hasattr(fd, 'read'): # Not a file, is it a function? fd = fd() safe_assert(hasattr(fd, 'read')) except (TypeError, AssertionError): return None message = email.parser.Parser().parse(fd) msi = message.signature_info = SignatureInfo(bubbly=False) mei = message.encryption_info = EncryptionInfo(bubbly=False) for part in message.walk(): part.signature_info = SignatureInfo(parent=msi) part.encryption_info = EncryptionInfo(parent=mei) if cache_id is not None: with GLOBAL_PARSE_CACHE_LOCK: # Keep 25 items, put new ones at the front GLOBAL_PARSE_CACHE[24:] = [] GLOBAL_PARSE_CACHE[:0] = [(cache_id, pgpmime, message)] return message def GetTextPayload(part): mimetype = part.get_content_type() or 'text/plain' cte = part.get('content-transfer-encoding', '').lower() if mimetype[:5] == 'text/' and cte == 'base64': # Mailing lists like to mess with text/plain parts, and Majordomo # in particular isn't aware of base64 encoding. Compensate! payload = part.get_payload(None, False) or '' parts = payload.split('\n--') try: parts[0] = base64.b64decode(parts[0]) except TypeError: pass return '\n--'.join(parts) else: return part.get_payload(None, True) or '' def ExtractEmails(string, strip_keys=True): emails = [] startcrap = re.compile('^[\'\"<(]') endcrap = re.compile('[\'\">);]$') string = string.replace('<', ' <').replace('(', ' (') for w in [sw.strip() for sw in re.compile('[,\s]+').split(string)]: atpos = w.find('@') if atpos >= 0: while startcrap.search(w): w = w[1:] while endcrap.search(w): w = w[:-1] if w.startswith('mailto:'): w = w[7:] if '?' in w: w = w.split('?')[0] if strip_keys and '#' in w[atpos:]: w = w[:atpos] + w[atpos:].split('#', 1)[0] # E-mail addresses are only allowed to contain ASCII # characters, so we just strip everything else away. emails.append(CleanText(w, banned=CleanText.WHITESPACE, replace='_').clean) return emails def ExtractEmailAndName(string): email = (ExtractEmails(string) or [''])[0] name = (string .replace(email, '') .replace('<>', '') .replace('"', '') .replace('(', '') .replace(')', '')).strip() return email, (name or email) def CleanHeaders(msg, copy_all=True, tombstones=False): clean_headers = [] address_headers_lower = [h.lower() for h in Email.ADDRESS_HEADERS] for key, value in msg.items(): lkey = key.lower() # Remove headers we don't want to expose if (lkey.startswith('x-mp-internal-') or lkey in ('bcc', 'encryption', 'attach-pgp-pubkey')): if tombstones: clean_headers.append((key, None)) # Strip the #key part off any e-mail addresses: elif lkey in address_headers_lower: if '#' in value: clean_headers.append((key, re.sub( r'(@[^<>\s#]+)#[a-fxA-F0-9]+([>,\s]|$)', r'\1\2', value))) elif copy_all: clean_headers.append((key, value)) elif copy_all: clean_headers.append((key, value)) return clean_headers def CleanMessage(config, msg): replacements = CleanHeaders(msg, copy_all=False, tombstones=True) for key, val in replacements: del msg[key] for key, val in replacements: if val: msg[key] = val return msg def PrepareMessage(config, msg, sender=None, rcpts=None, events=None, bounce=False): msg = copy.deepcopy(msg) # Short circuit if this message has already been prepared. if ('x-mp-internal-sender' in msg and 'x-mp-internal-rcpts' in msg and not bounce): return (sender or msg['x-mp-internal-sender'], rcpts or [r.strip() for r in msg['x-mp-internal-rcpts'].split(',')], msg, events) crypto_policy = 'default' crypto_format = 'default' rcpts = rcpts or [] if bounce: safe_assert(len(rcpts) > 0) # Iterate through headers to figure out what we want to do... need_rcpts = not rcpts for hdr, val in msg.items(): lhdr = hdr.lower() if lhdr == 'from': sender = sender or val elif lhdr == 'encryption': crypto_policy = val elif need_rcpts and lhdr in ('to', 'cc', 'bcc'): rcpts += AddressHeaderParser(val).addresses_list(with_keys=True) # Are we sane? if not sender: raise NoFromAddressError() if not rcpts: raise NoRecipientError() # Are we encrypting? Signing? crypto_policy = crypto_policy.lower() if crypto_policy == 'default': crypto_policy = config.prefs.crypto_policy.lower() sender = AddressHeaderParser(sender)[0].address # FIXME: Shouldn't this be using config.get_profile instead? profile = config.vcards.get_vcard(sender) if profile: crypto_format = (profile.crypto_format or crypto_format).lower() if crypto_format == 'default': crypto_format = 'prefer_inline' if config.prefs.inline_pgp else '' # Extract just the e-mail addresses from the RCPT list, make unique rcpts, rr = [], rcpts for r in rr: for e in AddressHeaderParser(r).addresses_list(with_keys=True): if e not in rcpts: rcpts.append(e) # Bouncing disables all transformations, including crypto. if not bounce: # This is the BCC hack that Brennan hates! if config.prefs.always_bcc_self and sender not in rcpts: rcpts += [sender] # Add headers we require while 'date' in msg: del msg['date'] msg['Date'] = MakeMessageDate() import mailpile.plugins plugins = mailpile.plugins.PluginManager() # Perform pluggable content transformations sender, rcpts, msg, junk = plugins.outgoing_email_content_transform( config, sender, rcpts, msg) # Perform pluggable encryption transformations sender, rcpts, msg, matched = plugins.outgoing_email_crypto_transform( config, sender, rcpts, msg, crypto_policy=crypto_policy, crypto_format=crypto_format, cleaner=lambda m: CleanMessage(config, m)) if crypto_policy and (crypto_policy != 'none') and not matched: raise ValueError(_('Unknown crypto policy: %s') % crypto_policy) rcpts = set([r.rsplit('#', 1)[0] for r in rcpts]) msg['x-mp-internal-readonly'] = str(int(time.time())) msg['x-mp-internal-sender'] = sender msg['x-mp-internal-rcpts'] = ', '.join(rcpts) return (sender, rcpts, msg, events) class Email(object): """This is a lazy-loading object representing a single email.""" def __init__(self, idx, msg_idx_pos, msg_parsed=None, msg_parsed_pgpmime=(None, None), msg_info=None, ephemeral_mid=None): self.index = idx self.config = idx.config self.msg_idx_pos = msg_idx_pos self.ephemeral_mid = ephemeral_mid self.reset_caches(msg_parsed=msg_parsed, msg_parsed_pgpmime=msg_parsed_pgpmime, msg_info=msg_info, clear_parse_cache=False) def msg_mid(self): return self.ephemeral_mid or b36(self.msg_idx_pos) @classmethod def encoded_hdr(self, msg, hdr, value=None): hdr_value = value or (msg and msg.get(hdr)) or '' try: hdr_value.encode('us-ascii') except (UnicodeEncodeError, UnicodeDecodeError): if hdr.lower() in ('from', 'to', 'cc', 'bcc'): addrs = [] for addr in [a.strip() for a in hdr_value.split(',')]: name, part = [], [] words = addr.split() for w in words: if w[0] == '<' or '@' in w: part.append((w, 'us-ascii')) else: name.append(w) if name: name = ' '.join(name) try: part[0:0] = [(name.encode('us-ascii'), 'us-ascii')] except: part[0:0] = [(name, 'utf-8')] addrs.append(email.header.make_header(part).encode()) hdr_value = ', '.join(addrs) else: parts = [(hdr_value, 'utf-8')] hdr_value = email.header.make_header(parts).encode() return hdr_value @classmethod def Create(cls, idx, mbox_id, mbx, msg_to=None, msg_cc=None, msg_bcc=None, msg_from=None, msg_subject=None, msg_text='', msg_references=None, msg_id=None, msg_atts=None, msg_headers=None, save=True, ephemeral_mid='not-saved', append_sig=True, use_default_from=True): msg = MIMEMultipart(boundary=MakeBoundary()) msg.signature_info = msi = SignatureInfo(bubbly=False) msg.encryption_info = mei = EncryptionInfo(bubbly=False) msg_ts = int(time.time()) if msg_from: from_email = AddressHeaderParser(unicode_data=msg_from)[0].address from_profile = idx.config.get_profile(email=from_email) elif use_default_from: from_profile = idx.config.get_profile() from_email = from_profile.get('email', None) from_name = from_profile.get('name', None) if from_email and from_name: msg_from = '%s <%s>' % (from_name, from_email) else: from_email = from_profile = from_name = None if msg_from: msg['From'] = cls.encoded_hdr(None, 'from', value=msg_from) msg['Date'] = MakeMessageDate(msg_ts) msg['Message-Id'] = msg_id or MakeMessageID() msg_subj = (msg_subject or '') msg['Subject'] = cls.encoded_hdr(None, 'subject', value=msg_subj) # Privacy trade-off: we want to help recipients do profiling and # discard poorly forged messages that are not from from Mailpile. # However, we don't want to leak too many details for privacy and # security reasons. So no: version or platform info, just the word # Mailpile. This will probably be obvious to a truly hostile # adversary anyway from other details. msg['User-Agent'] = 'Mailpile' ahp = AddressHeaderParser() norm = lambda a: ', '.join(sorted(list(set(ahp.normalized_addresses( addresses=a, with_keys=True, force_name=True))))) if msg_to: msg['To'] = cls.encoded_hdr(None, 'to', value=norm(msg_to)) if msg_cc: msg['Cc'] = cls.encoded_hdr(None, 'cc', value=norm(msg_cc)) if msg_bcc: msg['Bcc'] = cls.encoded_hdr(None, 'bcc', value=norm(msg_bcc)) if msg_references: msg['In-Reply-To'] = msg_references[-1] msg['References'] = ', '.join(msg_references) if msg_text: try: msg_text.encode('us-ascii') charset = 'us-ascii' except (UnicodeEncodeError, UnicodeDecodeError): charset = 'utf-8' tp = MIMEText(msg_text, _subtype='plain', _charset=charset) tp.signature_info = SignatureInfo(parent=msi) tp.encryption_info = EncryptionInfo(parent=mei) msg.attach(tp) del tp['MIME-Version'] for k, v in (msg_headers or []): msg[k] = v if msg_atts: for att in msg_atts: att = copy.deepcopy(att) att.signature_info = SignatureInfo(parent=msi) att.encryption_info = EncryptionInfo(parent=mei) # Disabled for now. # if att.get('content-id') is None: # att.add_header('Content-Id', MakeContentID()) msg.attach(att) del att['MIME-Version'] # Determine if we want to attach a PGP public key due to policy and # timing... if (idx.config.prefs.gpg_email_key and from_profile and 'send_keys' in from_profile.get('crypto_format', 'none')): from mailpile.plugins.crypto_policy import CryptoPolicy addrs = ExtractEmails(norm(msg_to) + norm(msg_cc) + norm(msg_bcc)) if CryptoPolicy.ShouldAttachKey(idx.config, emails=addrs): msg["Attach-PGP-Pubkey"] = "Yes" if save: msg_key = mbx.add(MessageAsString(msg)) msg_to = msg_cc = [] msg_ptr = mbx.get_msg_ptr(mbox_id, msg_key) msg_id = idx.get_msg_id(msg, msg_ptr) msg_idx, msg_info = idx.add_new_msg(msg_ptr, msg_id, msg_ts, msg_from, msg_to, msg_cc, 0, msg_subj, '', []) idx.set_conversation_ids(msg_info[idx.MSG_MID], msg, subject_threading=False) return cls(idx, msg_idx) else: msg_info = idx.edit_msg_info(idx.BOGUS_METADATA[:], msg_mid=ephemeral_mid or '', msg_id=msg['Message-ID'], msg_ts=msg_ts, msg_subject=msg_subj, msg_from=msg_from, msg_to=msg_to, msg_cc=msg_cc) return cls(idx, -1, msg_info=msg_info, msg_parsed=msg, msg_parsed_pgpmime=('basic', msg), ephemeral_mid=ephemeral_mid) def is_editable(self, quick=False): if self.ephemeral_mid: return True if not self.config.is_editable_message(self.get_msg_info()): return False if quick: return True return ('x-mp-internal-readonly' not in self.get_msg(pgpmime=False)) MIME_HEADERS = ('mime-version', 'content-type', 'content-disposition', 'content-transfer-encoding') UNEDITABLE_HEADERS = ('message-id', ) + MIME_HEADERS MANDATORY_HEADERS = ('From', 'To', 'Cc', 'Bcc', 'Subject', 'Encryption', 'Attach-PGP-Pubkey') ADDRESS_HEADERS = ('From', 'To', 'Cc', 'Bcc', 'Reply-To') HEADER_ORDER = { 'in-reply-to': -2, 'references': -1, 'date': 1, 'from': 2, 'subject': 3, 'to': 4, 'cc': 5, 'bcc': 6, 'encryption': 98, 'attach-pgp-pubkey': 99, } def _attachment_aid(self, att): aid = att.get('aid') if not aid: cid = att.get('content-id') # This comes from afar and might # be malicious, so check it. if (cid and cid == CleanText(cid, banned=(CleanText.WHITESPACE + CleanText.FS)).clean): aid = cid else: aid = 'part-%s' % att['count'] return aid def get_editing_strings(self, tree=None, build_tree=True): if build_tree: tree = self.get_message_tree(want=['editing_strings'], tree=tree) strings = { 'from': '', 'to': '', 'cc': '', 'bcc': '', 'subject': '', 'encryption': '', 'attach-pgp-pubkey': '', 'attachments': {} } header_lines = [] body_lines = [] # We care about header order and such things... hdrs = dict([(h.lower(), h) for h in tree['headers'].keys() if h.lower() not in self.UNEDITABLE_HEADERS]) for mandate in self.MANDATORY_HEADERS: hdrs[mandate.lower()] = hdrs.get(mandate.lower(), mandate) keys = hdrs.keys() keys.sort(key=lambda k: (self.HEADER_ORDER.get(k.lower(), 99), k)) lowman = [m.lower() for m in self.MANDATORY_HEADERS] lowadr = [m.lower() for m in self.ADDRESS_HEADERS] for hdr in [hdrs[k] for k in keys]: data = tree['headers'].get(hdr, '') lhdr = hdr.lower() if lhdr in lowadr and lhdr in lowman: adata = tree.get('addresses', {}).get(lhdr, None) if adata is None: adata = AddressHeaderParser(data) strings[lhdr] = adata.normalized() elif lhdr in lowman: strings[lhdr] = unicode(data) else: header_lines.append(unicode('%s: %s' % (hdr, data))) for att in tree['attachments']: aid = self._attachment_aid(att) strings['attachments'][aid] = (att['filename'] or '(unnamed)') if not strings['encryption']: strings['encryption'] = unicode(self.config.prefs.crypto_policy) def _fixup(t): try: return unicode(t) except UnicodeDecodeError: return t.decode('utf-8') strings['headers'] = '\n'.join(header_lines).replace('\r\n', '\n') strings['body'] = unicode(''.join([_fixup(t['data']) for t in tree['text_parts']]) ).replace('\r\n', '\n') return strings def get_editing_string(self, tree=None, estrings=None, attachment_headers=True, build_tree=True): if estrings is None: estrings = self.get_editing_strings(tree=tree, build_tree=build_tree) bits = [estrings['headers']] if estrings['headers'] else [] for mh in self.MANDATORY_HEADERS: bits.append('%s: %s' % (mh, estrings[mh.lower()])) if attachment_headers: for aid in sorted(estrings['attachments'].keys()): bits.append('Attachment-%s: %s' % (aid, estrings['attachments'][aid])) bits.append('') bits.append(estrings['body']) return '\n'.join(bits) def _update_att_name(self, part, filename): try: del part['Content-Disposition'] except KeyError: pass part.add_header('Content-Disposition', 'attachment', filename=filename) return part def _make_attachment(self, fn, msg, filedata=None): if filedata and fn in filedata: data = filedata[fn] else: if isinstance(fn, unicode): fn = fn.encode('utf-8') data = open(fn, 'rb').read() ctype, encoding = mimetypes.guess_type(fn) maintype, subtype = (ctype or 'application/octet-stream').split('/', 1) if maintype == 'image': att = MIMEImage(data, _subtype=subtype) else: att = MIMEBase(maintype, subtype) att.set_payload(data) encoders.encode_base64(att) # Disabled for now. # att.add_header('Content-Id', MakeContentID()) # FS paths are strings of bytes, should be represented as utf-8 for # correct header encoding. base_fn = os.path.basename(fn) if not isinstance(base_fn, unicode): base_fn = base_fn.decode('utf-8') att.add_header('Content-Disposition', 'attachment', filename=self.encoded_hdr(None, 'file', base_fn)) att.signature_info = SignatureInfo(parent=msg.signature_info) att.encryption_info = EncryptionInfo(parent=msg.encryption_info) return att def update_from_string(self, session, data, final=False): if not self.is_editable(): raise NotEditableError(_('Message or mailbox is read-only.')) oldmsg = self.get_msg() if not data: outmsg = oldmsg else: newmsg = email.parser.Parser().parsestr(data.encode('utf-8')) outmsg = MIMEMultipart(boundary=MakeBoundary()) outmsg.signature_info = SignatureInfo(bubbly=False) outmsg.encryption_info = EncryptionInfo(bubbly=False) # Copy over editable headers from the input string, skipping blanks for hdr in newmsg.keys(): if hdr.startswith('Attachment-') or hdr == 'Attachment': pass else: encoded_hdr = self.encoded_hdr(newmsg, hdr) if len(encoded_hdr.strip()) > 0: if encoded_hdr == '!KEEP': if hdr in oldmsg: outmsg[hdr] = oldmsg[hdr] else: outmsg[hdr] = encoded_hdr # Copy over the uneditable headers from the old message for hdr in oldmsg.keys(): if ((hdr.lower() not in self.MIME_HEADERS) and (hdr.lower() in self.UNEDITABLE_HEADERS)): outmsg[hdr] = oldmsg[hdr] # Copy the message text new_body = newmsg.get_payload().decode('utf-8') target_width = self.config.prefs.line_length if target_width >= 40 and 'x-mp-internal-no-reflow' not in newmsg: new_body = reflow_text(new_body, target_width=target_width) try: new_body.encode('us-ascii') charset = 'us-ascii' except (UnicodeEncodeError, UnicodeDecodeError): charset = 'utf-8' tp = MIMEText(new_body, _subtype='plain', _charset=charset) tp.signature_info = SignatureInfo(parent=outmsg.signature_info) tp.encryption_info = EncryptionInfo(parent=outmsg.encryption_info) outmsg.attach(tp) del tp['MIME-Version'] # FIXME: Use markdown and template to generate fancy HTML part? # Copy the attachments we are keeping attachments = [h for h in newmsg.keys() if h.lower().startswith('attachment')] if attachments: oldtree = self.get_message_tree(want=['attachments']) for att in oldtree['attachments']: hdr = 'Attachment-%s' % self._attachment_aid(att) if hdr in attachments: outmsg.attach(self._update_att_name(att['part'], newmsg[hdr])) attachments.remove(hdr) # Attach some new files? for hdr in attachments: try: att = self._make_attachment(newmsg[hdr], outmsg) outmsg.attach(att) del att['MIME-Version'] except: pass # FIXME: Warn user that failed... # Save result back to mailbox if final: sender, rcpts, outmsg, ev = PrepareMessage(self.config, outmsg) return self.update_from_msg(session, outmsg) def update_from_msg(self, session, newmsg): if not self.is_editable(): raise NotEditableError(_('Message or mailbox is read-only.')) if self.ephemeral_mid: self.reset_caches(clear_parse_cache=False, msg_parsed=newmsg, msg_parsed_pgpmime=('basic', newmsg), msg_info=self.msg_info) else: mbx, ptr, fd = self.get_mbox_ptr_and_fd() fd.close() # Windows needs this # OK, adding to the mailbox worked newptr = ptr[:MBX_ID_LEN] + mbx.add(MessageAsString(newmsg)) self.update_parse_cache(newmsg) # Remove the old message... mbx.remove_by_ptr(ptr) # FIXME: We should DELETE the old version from the index first. # Update the in-memory-index mi = self.get_msg_info() mi[self.index.MSG_PTRS] = newptr self.index.set_msg_at_idx_pos(self.msg_idx_pos, mi) self.index.index_email(session, Email(self.index, self.msg_idx_pos)) self.reset_caches(clear_parse_cache=False) return self def reset_caches(self, msg_info=None, msg_parsed=None, msg_parsed_pgpmime=(None, None), clear_parse_cache=True): self.msg_info = msg_info self.msg_parsed = msg_parsed self.msg_parsed_pgpmime = msg_parsed_pgpmime if clear_parse_cache: self.clear_from_parse_cache() def update_parse_cache(self, newmsg): cache_id = self.get_cache_id() if cache_id: with GLOBAL_PARSE_CACHE_LOCK: GPC = GLOBAL_PARSE_CACHE for i in range(0, len(GPC)): if GPC[i][0] == cache_id: GPC[i] = (cache_id, False, newmsg) def clear_from_parse_cache(self): cache_id = self.get_cache_id() if cache_id: ClearParseCache(cache_id=cache_id) def delete_message(self, session, flush=True, keep=0): mi = self.get_msg_info() removed, failed, mailboxes = [], [], [] kept = keep allow_deletion = session.config.prefs.allow_deletion for msg_ptr, mbox, fd in self.index.enumerate_ptrs_mboxes_fds(mi): try: if mbox: try: if keep > 0: # Note: This will keep messages in the order of # preference implemented by enumerate_ptrs_... # FIXME: Allow more nuanced behaviour here. mbox.get_file_by_ptr(msg_ptr) keep -= 1 elif allow_deletion: mbox.remove_by_ptr(msg_ptr) else: # FIXME: Allow deletion of local copies ONLY raise ValueError("Deletion is forbidden") except (KeyError, IndexError): # Already gone! pass mailboxes.append(mbox) removed.append(msg_ptr) except (IOError, OSError, ValueError, AttributeError) as e: failed.append(msg_ptr) print('FIXME: Could not delete %s: %s' % (msg_ptr, e)) if allow_deletion and not failed and not kept: self.index.delete_msg_at_idx_pos(session, self.msg_idx_pos, keep_msgid=False) if flush: for m in mailboxes: m.flush() return (not failed, []) else: return (not failed, mailboxes) def get_msg_info(self, field=None, uncached=False): if (uncached or not self.msg_info) and not self.ephemeral_mid: self.msg_info = self.index.get_msg_at_idx_pos(self.msg_idx_pos) if field is None: return self.msg_info else: return self.msg_info[field] def get_mbox_ptr_and_fd(self): mi = self.get_msg_info() for msg_ptr, mbox, fd in self.index.enumerate_ptrs_mboxes_fds(mi): if fd is not None: # FIXME: How do we know we have the right message? return mbox, msg_ptr, FixupForWith(fd) return None, None, None def get_file(self): return self.get_mbox_ptr_and_fd()[2] def get_msg_size(self): mbox, ptr, fd = self.get_mbox_ptr_and_fd() with fd: fd.seek(0, 2) return fd.tell() def get_metadata_kws(self): # FIXME: Track these somehow... return [] def get_cache_id(self): if (self.msg_idx_pos >= 0) and not self.ephemeral_mid: return '%s/%s' % (self.index, self.msg_idx_pos) else: return None def _get_parsed_msg(self, pgpmime, update_cache=False): weak_crypto_max_age = self.config.prefs.weak_crypto_max_age allow_weak_crypto = False if weak_crypto_max_age > 0: ts = int(self.get_msg_info(self.index.MSG_DATE) or '0', 36) allow_weak_crypto = (ts < weak_crypto_max_age) return ParseMessage(self.get_file, cache_id=self.get_cache_id(), update_cache=update_cache, pgpmime=pgpmime, config=self.config, allow_weak_crypto=allow_weak_crypto, event=GetThreadEvent()) def _update_crypto_state(self): if not (self.config.tags and self.msg_idx_pos >= 0 and self.msg_parsed_pgpmime[0] and self.msg_parsed_pgpmime[1] and not self.ephemeral_mid): return import mailpile.plugins.cryptostate as cs kw = cs.meta_kw_extractor(self.index, self.msg_mid(), self.msg_parsed_pgpmime[1], 0, 0) # msg_size, msg_ts # We do NOT want to update tags if we are getting back # a none/none state, as that can happen for the more # complex nested crypto-in-text messages, which a more # forceful parse of the message may have caught earlier. no_sig = self.config.get_tag('mp_sig-none') no_sig = no_sig and '%s:in' % no_sig._key no_enc = self.config.get_tag('mp_enc-none') no_enc = no_enc and '%s:in' % no_enc._key if no_sig not in kw or no_enc not in kw: msg_info = self.get_msg_info() msg_tags = msg_info[self.index.MSG_TAGS].split(',') msg_tags = sorted([t for t in msg_tags if t]) # Note: this has the side effect of cleaning junk off # the tag list, not just updating crypto state. def tcheck(tag_id): tag = self.config.get_tag(tag_id) return (tag and tag.slug[:6] not in ('mp_enc', 'mp_sig')) new_tags = sorted([t for t in msg_tags if tcheck(t)] + [ti.split(':', 1)[0] for ti in kw if ti.endswith(':in')]) if msg_tags != new_tags: msg_info[self.index.MSG_TAGS] = ','.join(new_tags) self.index.set_msg_at_idx_pos(self.msg_idx_pos, msg_info) def get_msg(self, pgpmime='default', crypto_state_feedback=True): if pgpmime: if pgpmime == 'default': pgpmime = 'basic' if self.is_editable() else 'all' if self.msg_parsed_pgpmime[0] == pgpmime: result = self.msg_parsed_pgpmime[1] else: result = self._get_parsed_msg(pgpmime) self.msg_parsed_pgpmime = (pgpmime, result) # Post-parse, we want to make sure that the crypto-state # recorded on this message's metadata is up to date. if crypto_state_feedback: self._update_crypto_state() else: if not self.msg_parsed: self.msg_parsed = self._get_parsed_msg(pgpmime) result = self.msg_parsed if not result: raise IndexError(_('Message not found')) return result def is_thread(self): return ((self.get_msg_info(self.index.MSG_THREAD_MID)) or (0 < len(self.get_msg_info(self.index.MSG_REPLIES)))) def get(self, field, default=''): """Get one (or all) indexed fields for this mail.""" field = field.lower() if field == 'subject': return self.get_msg_info(self.index.MSG_SUBJECT) elif field == 'from': return self.get_msg_info(self.index.MSG_FROM) else: raw = ' '.join(self.get_msg(pgpmime=False).get_all(field, default)) return safe_decode_hdr(hdr=raw) or raw def get_sender(self): try: ahp = AddressHeaderParser(unicode_data=self.get('from')) return ahp[0].address except IndexError: return None def get_headerprints(self): return HeaderPrints(self.get_msg(pgpmime='basic')) def get_msg_summary(self): # We do this first to make sure self.msg_info is loaded msg_mid = self.get_msg_info(self.index.MSG_MID) return [ msg_mid, self.get_msg_info(self.index.MSG_ID), self.get_msg_info(self.index.MSG_FROM), self.index.expand_to_list(self.msg_info), self.get_msg_info(self.index.MSG_SUBJECT), self.get_msg_info(self.index.MSG_BODY), self.get_msg_info(self.index.MSG_DATE), self.get_msg_info(self.index.MSG_TAGS).split(','), self.is_editable(quick=True) ] def _find_attachments(self, att_id, negative=False): msg = self.get_msg() count = 0 for part in (msg.walk() if msg else []): mimetype = (part.get_content_type() or 'text/plain').lower() if mimetype.startswith('multipart/'): continue count += 1 content_id = part.get('content-id', '') pfn = safe_decode_hdr(hdr=part.get_filename() or '') if (('*' == att_id) or ('#%s' % count == att_id) or ('part-%s' % count == att_id) or (content_id == att_id) or (mimetype == att_id) or (pfn.lower().endswith('.%s' % att_id)) or (pfn == att_id)): if not negative: yield (count, content_id, pfn, mimetype, part) elif negative: yield (count, content_id, pfn, mimetype, part) def add_attachments(self, session, filenames, filedata=None): if not self.is_editable(): raise NotEditableError(_('Message or mailbox is read-only.')) msg = self.get_msg() for fn in filenames: att = self._make_attachment(fn, msg, filedata=filedata) msg.attach(att) del att['MIME-Version'] return self.update_from_msg(session, msg) def remove_attachments(self, session, *att_ids): if not self.is_editable(): raise NotEditableError(_('Message or mailbox is read-only.')) remove = [] for att_id in att_ids: for count, cid, pfn, mt, part in self._find_attachments(att_id): remove.append(self._attachment_aid({ 'msg_mid': self.msg_mid(), 'count': count, 'content-id': cid, 'filename': pfn, })) es = self.get_editing_strings() es['headers'] = None for k in remove: if k in es['attachments']: del es['attachments'][k] estring = self.get_editing_string(estrings=es) return self.update_from_string(session, estring) def extract_attachment(self, session, att_id, name_fmt=None, mode='get'): extracted = 0 filename, attributes = '', {} for (count, content_id, pfn, mimetype, part ) in self._find_attachments(att_id): payload = part.get_payload(None, True) or '' attributes = { 'msg_mid': self.msg_mid(), 'count': count, 'length': len(payload), 'content-id': content_id, 'filename': pfn} attributes['aid'] = self._attachment_aid(attributes) if pfn: if '.' in pfn: pfn, attributes['att_ext'] = pfn.rsplit('.', 1) attributes['att_ext'] = attributes['att_ext'].lower() attributes['att_name'] = pfn if mimetype: attributes['mimetype'] = mimetype filesize = len(payload) if mode.startswith('inline'): attributes['data'] = payload session.ui.notify(_('Extracted attachment %s') % att_id) elif mode.startswith('preview'): attributes['thumb'] = True attributes['mimetype'] = 'image/jpeg' attributes['disposition'] = 'inline' thumb = StringIO.StringIO() if thumbnail(payload, thumb, height=250): attributes['length'] = thumb.tell() filename, fd = session.ui.open_for_data( name_fmt=name_fmt, attributes=attributes) thumb.seek(0) fd.write(thumb.read()) fd.close() session.ui.notify(_('Wrote preview to: %s') % filename) else: session.ui.notify(_('Failed to generate thumbnail')) raise UrlRedirectException('/static/img/image-default.png') else: WHITELIST = ('image/png', 'image/gif', 'image/jpeg', 'image/tiff', 'audio/mp3', 'audio/ogg', 'audio/x-wav', 'audio/mpeg', 'video/mpeg', 'video/ogg', 'application/pdf') if mode.startswith('get') and mimetype in WHITELIST: # This allows the browser to (optionally) handle the # content, instead of always forcing a download dialog. attributes['disposition'] = 'inline' filename, fd = session.ui.open_for_data( name_fmt=name_fmt, attributes=attributes) fd.write(payload) session.ui.notify(_('Wrote attachment to: %s') % filename) fd.close() extracted += 1 if 0 == extracted: session.ui.notify(_('No attachments found for: %s') % att_id) return None, None else: return filename, attributes def get_message_tags(self): tids = self.get_msg_info(self.index.MSG_TAGS).split(',') return [self.config.get_tag(t) for t in tids] def get_message_tree(self, want=None, tree=None, pgpmime='default'): msg = self.get_msg(pgpmime=pgpmime) want = list(want) if (want is not None) else None tree = tree or {'_cleaned': []} tree['id'] = self.get_msg_info(self.index.MSG_ID) if want is not None: if 'editing_strings' in want or 'editing_string' in want: want.extend(['text_parts', 'headers', 'attachments', 'addresses']) for p in 'text_parts', 'html_parts', 'vcal_parts', 'attachments': if want is None or p in want: tree[p] = [] if (want is None or 'summary' in want) and 'summary' not in tree: tree['summary'] = self.get_msg_summary() if (want is None or 'tags' in want) and 'tags' not in tree: tree['tags'] = self.get_msg_info(self.index.MSG_TAGS).split(',') if (want is None or 'conversation' in want ) and 'conversation' not in tree: tree['conversation'] = {} conv_id = self.get_msg_info(self.index.MSG_THREAD_MID) if conv_id: conv_id = conv_id.split('/')[0] conv = Email(self.index, int(conv_id, 36)) tree['conversation'] = convs = [conv.get_msg_summary()] for rid in conv.get_msg_info(self.index.MSG_REPLIES ).split(','): if rid: convs.append(Email(self.index, int(rid, 36) ).get_msg_summary()) if (want is None or 'headerprints' in want): tree['headerprints'] = self.get_headerprints() if (want is None or 'headers' in want) and 'headers' not in tree: tree['headers'] = {} for hdr in msg.keys(): tree['headers'][hdr] = safe_decode_hdr(msg, hdr) if (want is None or 'headers_lc' in want ) and 'headers_lc' not in tree: tree['headers_lc'] = {} for hdr in msg.keys(): tree['headers_lc'][hdr.lower()] = safe_decode_hdr(msg, hdr) if (want is None or 'header_list' in want ) and 'header_list' not in tree: tree['header_list'] = [(k, safe_decode_hdr(msg, k, hdr=v)) for k, v in msg.items()] if (want is None or 'addresses' in want ) and 'addresses' not in tree: address_headers_lower = [h.lower() for h in self.ADDRESS_HEADERS] tree['addresses'] = {} for hdr in msg.keys(): hdrl = hdr.lower() if hdrl in address_headers_lower: tree['addresses'][hdrl] = AddressHeaderParser(msg[hdr]) # Note: count algorithm must match that used in extract_attachment # above count = 0 broken_text_part = None for part in msg.walk(): crypto = { 'signature': part.signature_info, 'encryption': part.encryption_info} mimetype = (part.get_content_type() or 'text/plain').lower() if (mimetype.startswith('multipart/') or mimetype == "application/pgp-encrypted"): continue try: if (mimetype == "application/octet-stream" and part.cryptedcontainer is True): continue except: pass count += 1 disposition = part.get('content-disposition', 'inline').lower() if (disposition[:6] == 'inline' and mimetype.startswith('text/')): payload, charset = self.decode_payload(part) start = payload[:100].strip() if mimetype == 'text/html': if want is None or 'html_parts' in want: tree['html_parts'].append({ 'charset': charset, 'type': 'html', 'data': clean_html(payload), 'count': count, 'mimetype': mimetype, 'aid': 'part-%d' % count}) elif mimetype == "text/calendar": if want is None or 'vcal_parts' in want: tree["vcal_parts"].extend(calendar_parse(payload)) elif want is None or 'text_parts' in want: for ht in ('', '

2 and '@' in clines[-2] and '' == clines[-1].strip()): current['data'] = ''.join(clines[:-2]) clines = clines[-2:] else: clines = [] current = { 'type': ltype, 'data': ''.join(clines), 'charset': charset, 'crypto': { 'signature': SignatureInfo(parent=psi), 'encryption': EncryptionInfo(parent=pei)}} parse.append(current) if len(parse) == 1 and count and mimetype: current['aid'] = 'part-%d' % count current['mimetype'] = mimetype current['data'] += line clines.append(line) return parse BARE_QUOTE_STARTS = re.compile('(?i)^-+\s*Original Message.*-+$') GIT_DIFF_STARTS = re.compile('^diff --git a/.*b/') GIT_DIFF_LINE = re.compile('^([ +@-]|index |$)') def parse_line_type(self, line, block, line_count): # FIXME: Detect forwarded messages, ... if (block in ('body', 'quote', 'barequote') and line in ('-- \n', '-- \r\n', '- --\n', '- --\r\n')): return 'signature', 'signature' if block == 'signature': return block, block if block == 'barequote': return 'barequote', 'quote' stripped = line.rstrip() if stripped == GnuPG.ARMOR_BEGIN_SIGNED: return 'pgpbeginsigned', 'pgpbeginsigned' if block == 'pgpbeginsigned': if line.startswith('Hash: ') or stripped == '': return 'pgpbeginsigned', 'pgpbeginsigned' else: return 'pgpsignedtext', 'pgpsignedtext' if block == 'pgpsignedtext': if stripped == GnuPG.ARMOR_BEGIN_SIGNATURE: return 'pgpsignature', 'pgpsignature' else: return 'pgpsignedtext', 'pgpsignedtext' if block == 'pgpsignature': if stripped == GnuPG.ARMOR_END_SIGNATURE: return 'pgpend', 'pgpsignature' else: return 'pgpsignature', 'pgpsignature' if (stripped == GnuPG.ARMOR_BEGIN_ENCRYPTED # This is an EFail mitigation: do not decrypt content # inlined somewhere well below a bunch of other stuff. # The encrypted content must be high up enough that # the user will plausibly see it when reading. and line_count < 10 and block == 'body'): return 'pgpbegin', 'pgpbegin' if block == 'pgpbegin': if ':' in line or stripped == '': return 'pgpbegin', 'pgpbegin' else: return 'pgptext', 'pgptext' if block == 'pgptext': if stripped == GnuPG.ARMOR_END_ENCRYPTED: return 'pgpend', 'pgpend' else: return 'pgptext', 'pgptext' if self.BARE_QUOTE_STARTS.match(stripped): return 'barequote', 'quote' if block == 'quote': if stripped == '': return 'quote', 'quote' if line.startswith('>'): return 'quote', 'quote' if self.GIT_DIFF_STARTS.match(stripped): return 'gitdiff', 'quote' if block == 'gitdiff': if self.GIT_DIFF_LINE.match(stripped): return 'gitdiff', 'quote' return 'body', 'text' WANT_MSG_TREE_PGP = ('text_parts', 'crypto') PGP_OK = { 'pgpbeginsigned': 'pgpbeginverified', 'pgpsignedtext': 'pgpverifiedtext', 'pgpsignature': 'pgpverification', 'pgpbegin': 'pgpbeginverified', 'pgptext': 'pgpsecuretext', 'pgpend': 'pgpverification', } def evaluate_pgp(self, tree, check_sigs=True, decrypt=False, crypto_state_feedback=True, event=None): if 'text_parts' not in tree: return tree pgpdata = [] for part in tree['text_parts']: if 'crypto' not in part: part['crypto'] = {} ei = si = None if check_sigs: if part['type'] == 'pgpbeginsigned': pgpdata = [part] elif part['type'] == 'pgpsignedtext': pgpdata.append(part) elif part['type'] == 'pgpsignature': pgpdata.append(part) try: gpg = GnuPG(self.config, event=event) message = ''.join([p['data'].encode(p['charset']) for p in pgpdata]) si = gpg.verify(message) pgpdata[0]['data'] = '' pgpdata[1]['crypto']['signature'] = si pgpdata[2]['data'] = '' except Exception as e: print(e) if decrypt: if part['type'] in ('pgpbegin', 'pgptext'): pgpdata.append(part) elif part['type'] == 'pgpend': pgpdata.append(part) data = ''.join([p['data'] for p in pgpdata]) gpg = GnuPG(self.config, event=event) si, ei, text = gpg.decrypt(data) # FIXME: If the data is binary, we should provide some # sort of download link or maybe leave the PGP # blob entirely intact, undecoded. text, charset = self.decode_text(text, binary=False) pgpdata[1]['crypto']['encryption'] = ei pgpdata[1]['crypto']['signature'] = si if ei["status"] == "decrypted": pgpdata[0]['data'] = "" pgpdata[1]['data'] = text pgpdata[2]['data'] = "" # Bubbling up! if (si or ei) and 'crypto' not in tree: tree['crypto'] = {'signature': SignatureInfo(bubbly=False), 'encryption': EncryptionInfo(bubbly=False)} if si: si.bubble_up(tree['crypto']['signature']) if ei: ei.bubble_up(tree['crypto']['encryption']) # Cleanup, remove empty 'crypto': {} blocks. for part in tree['text_parts']: if not part['crypto']: del part['crypto'] tree['crypto']['signature'].mix_bubbles() tree['crypto']['encryption'].mix_bubbles() if crypto_state_feedback: self._update_crypto_state() return tree def _decode_gpg(self, message, decrypted): header, body = message.replace('\r\n', '\n').split('\n\n', 1) for line in header.lower().split('\n'): if line.startswith('charset:'): return decrypted.decode(line.split()[1]) return decrypted.decode('utf-8') if __name__ == "__main__": import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/generator.py ================================================ # Copyright (C) 2001-2010 Python Software Foundation # Contact: email-sig@python.org # # Updated/forked January 2014 by Bjarni R. Einarsson # to match the python 3.x email.generator CRLF control API (linesep=...). # # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 # -------------------------------------------- # # 1. This LICENSE AGREEMENT is between the Python Software Foundation # ("PSF"), and the Individual or Organization ("Licensee") accessing and # otherwise using this software ("Python") in source or binary form and # its associated documentation. # # 2. Subject to the terms and conditions of this License Agreement, PSF # hereby grants Licensee a nonexclusive, royalty-free, world-wide # license to reproduce, analyze, test, perform and/or display publicly, # prepare derivative works, distribute, and otherwise use Python # alone or in any derivative version, provided, however, that PSF's # License Agreement and PSF's notice of copyright, i.e., "Copyright (c) # 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights # Reserved" are retained in Python alone or in any derivative version # prepared by Licensee. # # 3. In the event Licensee prepares a derivative work that is based on # or incorporates Python or any part thereof, and wants to make # the derivative work available to others as provided herein, then # Licensee hereby agrees to include in any such work a brief summary of # the changes made to Python. # # 4. PSF is making Python available to Licensee on an "AS IS" # basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR # IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND # DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS # FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT # INFRINGE ANY THIRD PARTY RIGHTS. # # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON # FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS # A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, # OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. # # 6. This License Agreement will automatically terminate upon a material # breach of its terms and conditions. # # 7. Nothing in this License Agreement shall be deemed to create any # relationship of agency, partnership, or joint venture between PSF and # Licensee. This License Agreement does not grant permission to use PSF # trademarks or trade name in a trademark sense to endorse or promote # products or services of Licensee, or any third party. # # 8. By copying, installing or otherwise using Python, Licensee # agrees to be bound by the terms and conditions of this License # Agreement. """Classes to generate plain text from a message object tree.""" from __future__ import print_function __all__ = ['Generator', 'DecodedGenerator'] import re import sys import time import random import warnings from cStringIO import StringIO from email.header import Header from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n UNDERSCORE = '_' NL = '\n' fcre = re.compile(r'^From ', re.MULTILINE) def _is8bitstring(s): if isinstance(s, str): try: unicode(s, 'us-ascii') except UnicodeError: return True return False class Generator: """Generates output from a Message object tree. This basic generator writes the message to the given file object as plain text. """ # # Public interface # def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, linesep=None): """Create the generator for message flattening. outfp is the output file-like object for writing the message to. It must have a write() method. Optional mangle_from_ is a flag that, when True (the default), escapes From_ lines in the body of the message by putting a `>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) by RFC 2822. """ self._fp = outfp self._mangle_from_ = mangle_from_ self._maxheaderlen = maxheaderlen self._NL = linesep or NL def write(self, s): # Just delegate to the file object self._fp.write(s) def flatten(self, msg, unixfrom=False, linesep=None): """Print the message object tree rooted at msg to the output file specified when the Generator instance was created. unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message has no From_ delimiter, a `standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. linesep specifies the characters used to indicate a new line in the output. The default value is LF (not the standard CRLF). Note that for subobjects, no From_ line is printed. """ if linesep: self._NL = linesep if unixfrom: ufrom = msg.get_unixfrom() if not ufrom: ufrom = 'From nobody ' + time.ctime(time.time()) print(ufrom + self._NL, end='', file=self._fp) self._write(msg) def clone(self, fp): """Clone this generator with the exact same options.""" return self.__class__(fp, self._mangle_from_, self._maxheaderlen) # # Protected interface - undocumented ;/ # def _write(self, msg): # We can't write the headers yet because of the following scenario: # say a multipart message includes the boundary string somewhere in # its body. We'd have to calculate the new boundary /before/ we write # the headers so that we can write the correct Content-Type: # parameter. # # The way we do this, so as to make the _handle_*() methods simpler, # is to cache any subpart writes into a StringIO. The we write the # headers and the StringIO contents. That way, subpart handlers can # Do The Right Thing, and can still modify the Content-Type: header if # necessary. oldfp = self._fp try: self._fp = sfp = StringIO() self._dispatch(msg) finally: self._fp = oldfp # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. meth = getattr(msg, '_write_headers', None) if meth is None: self._write_headers(msg) else: meth(self) self._fp.write(sfp.getvalue()) def _dispatch(self, msg): # Get the Content-Type: for the message, then try to dispatch to # self._handle__(). If there's no handler for the # full MIME type, then dispatch to self._handle_(). If # that's missing too, then dispatch to self._writeBody(). main = msg.get_content_maintype() sub = msg.get_content_subtype() specific = UNDERSCORE.join((main, sub)).replace('-', '_') meth = getattr(self, '_handle_' + specific, None) if meth is None: generic = main.replace('-', '_') meth = getattr(self, '_handle_' + generic, None) if meth is None: meth = self._writeBody meth(msg) # # Default handlers # def _write_headers(self, msg): for h, v in msg.items(): print('%s:' % h, end=' ', file=self._fp) if self._maxheaderlen == 0: # Explicit no-wrapping print(v + self._NL, end='', file=self._fp) elif isinstance(v, Header): # Header instances know what to do hdr = v.encode().replace('\n', self._NL) print(hdr + self._NL, end='', file=self._fp) elif _is8bitstring(v): # If we have raw 8bit data in a byte string, we have no idea # what the encoding is. There is no safe way to split this # string. If it's ascii-subset, then we could do a normal # ascii split, but if it's multibyte then we could break the # string. There's no way to know so the least harm seems to # be to not split the string and risk it being too long. print(v + self._NL, end='', file=self._fp) else: # Header's got lots of smarts, so use it. Note that this is # fundamentally broken though because we lose idempotency when # the header string is continued with tabs. It will now be # continued with spaces. This was reversedly broken before we # fixed bug 1974. Either way, we lose. hdr = Header(v, maxlinelen=self._maxheaderlen, header_name=h ).encode().replace('\n', self._NL) print(hdr + self._NL, end='', file=self._fp) # A blank line always separates headers from body print(self._NL, end='', file=self._fp) # # Handlers for writing types and subtypes # def _handle_text(self, msg): payload = msg.get_payload() if payload is None: return if not isinstance(payload, basestring): raise TypeError('string payload expected: %s' % type(payload)) if self._mangle_from_: payload = fcre.sub('>From ', payload) self._fp.write(payload) # Default body handler _writeBody = _handle_text def _handle_multipart(self, msg): # The trick here is to write out each part separately, merge them all # together, and then make sure that the boundary we've chosen isn't # present in the payload. msgtexts = [] subparts = msg.get_payload() if subparts is None: subparts = [] elif isinstance(subparts, basestring): # e.g. a non-strict parse of a message with no starting boundary. self._fp.write(subparts) return elif not isinstance(subparts, list): # Scalar payload subparts = [subparts] for part in subparts: s = StringIO() g = self.clone(s) g.flatten(part, unixfrom=False, linesep=self._NL) msgtexts.append(s.getvalue()) # BAW: What about boundaries that are wrapped in double-quotes? boundary = msg.get_boundary() if not boundary: # Create a boundary that doesn't appear in any of the # message texts. alltext = self._NL.join(msgtexts) boundary = _make_boundary(alltext) msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: if self._mangle_from_: preamble = fcre.sub('>From ', msg.preamble) else: preamble = msg.preamble print(preamble + self._NL, end='', file=self._fp) # dash-boundary transport-padding CRLF print('--' + boundary + self._NL, end='', file=self._fp) # body-part if msgtexts: self._fp.write(msgtexts.pop(0)) # *encapsulation # --> delimiter transport-padding # --> CRLF body-part for body_part in msgtexts: # delimiter transport-padding CRLF print(self._NL + '--' + boundary + self._NL, end='', file=self._fp) # body-part self._fp.write(body_part) # close-delimiter transport-padding self._fp.write(self._NL + '--' + boundary + '--') if msg.epilogue is not None: print(self._NL, end='', file=self._fp) if self._mangle_from_: epilogue = fcre.sub('>From ', msg.epilogue) else: epilogue = msg.epilogue self._fp.write(epilogue) def _handle_multipart_signed(self, msg): # The contents of signed parts has to stay unmodified in order to keep # the signature intact per RFC1847 2.1, so we disable header wrapping. # RDM: This isn't enough to completely preserve the part, but it helps. # BRE: Disabled! We are using this to generate the stuff we sign, so # we actually want the logic UNCHANGED. return self._handle_multipart(msg) # Disabled the following... old_maxheaderlen = self._maxheaderlen try: self._maxheaderlen = 0 self._handle_multipart(msg) finally: self._maxheaderlen = old_maxheaderlen def _handle_message_delivery_status(self, msg): # We can't just write the headers directly to self's file object # because this will leave an extra newline between the last header # block and the boundary. Sigh. blocks = [] for part in msg.get_payload(): s = StringIO() g = self.clone(s) g.flatten(part, unixfrom=False, linesep=self._NL) text = s.getvalue() lines = text.split(self._NL) # Strip off the unnecessary trailing empty line if lines and lines[-1] == '': blocks.append(self._NL.join(lines[:-1])) else: blocks.append(text) # Now join all the blocks with an empty line. This has the lovely # effect of separating each block with an empty line, but not adding # an extra one after the last one. self._fp.write(self._NL.join(blocks)) def _handle_message(self, msg): s = StringIO() g = self.clone(s) # The payload of a message/rfc822 part should be a multipart sequence # of length 1. The zeroth element of the list should be the Message # object for the subpart. Extract that object, stringify it, and # write it out. # Except, it turns out, when it's a string instead, which happens when # and only when HeaderParser is used on a message of mime type # message/rfc822. Such messages are generated by, for example, # Groupwise when forwarding unadorned messages. (Issue 7970.) So # in that case we just emit the string body. payload = msg.get_payload() if isinstance(payload, list): g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) payload = s.getvalue() self._fp.write(payload) _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): """Generates a text representation of a message. Like the Generator base class, except that non-text parts are substituted with a format string representing the part. """ def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None, linesep=None): """Like Generator.__init__() except that an additional optional argument is allowed. Walks through all subparts of a message. If the subpart is of main type `text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in %(keyword)s format): type : Full MIME type of the non-text part maintype : Main MIME type of the non-text part subtype : Sub-MIME type of the non-text part filename : Filename of the non-text part description: Description associated with the non-text part encoding : Content transfer encoding of the non-text part The default value for fmt is None, meaning [Non-text (%(type)s) part of message omitted, filename %(filename)s] """ Generator.__init__(self, outfp, mangle_from_, maxheaderlen, linesep) if fmt is None: self._fmt = _FMT else: self._fmt = fmt def _dispatch(self, msg): for part in msg.walk(): maintype = part.get_content_maintype() if maintype == 'text': print(part.get_payload(decode=True) + self._NL, end='', file=self) elif maintype == 'multipart': # Just skip this pass else: print(self._fmt % { 'type': part.get_content_type(), 'maintype': part.get_content_maintype(), 'subtype': part.get_content_subtype(), 'filename': part.get_filename('[no filename]'), 'description': part.get('Content-Description', '[no description]'), 'encoding': part.get('Content-Transfer-Encoding', '[no encoding]'), } + self._NL, end='', file=self) # Helper _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width def _make_boundary(text=None): # Craft a random boundary. If text is given, ensure that the chosen # boundary doesn't appear in the text. token = random.randrange(sys.maxsize) boundary = ('=' * 15) + (_fmt % token) + '==' if text is None: return boundary b = boundary counter = 0 while True: cre = re.compile('^--' + re.escape(b) + '(--)?$', re.MULTILINE) if not cre.search(text): break b = boundary + '.' + str(counter) counter += 1 return b ================================================ FILE: mailpile/mailutils/header.py ================================================ # vim: set fileencoding=utf-8 : """ Backport of Python > 3.3 email.header.decode_header() It includes fixes that have not been ported to py2 https://bugs.python.org/issue1079 """ from __future__ import print_function import binascii import email.quoprimime import email.base64mime import re from email.errors import HeaderParseError # Match encoded-word strings in the form =?charset?q?Hello_World?= ecre = re.compile(r''' =\? # literal =? (?P[^?]*?) # non-greedy up to the next ? is the charset \? # literal ? (?P[qb]) # either a "q" or a "b", case insensitive \? # literal ? (?P.*?) # non-greedy up to the next ?= is the encoded string \?= # literal ?= ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) def decode_header(header): """ Decode a message header value without converting charset. Returns a list of (string, charset) pairs containing each of the decoded parts of the header. Charset is None for non-encoded parts of the header, otherwise a lower-case string containing the name of the character set specified in the encoded string. header may be a string that may or may not contain RFC2047 encoded words, or it may be a Header object. An email.errors.HeaderParseError may be raised when certain decoding error occurs (e.g. a base64 decoding exception). """ # If it is a Header object, we can just return the encoded chunks. if hasattr(header, '_chunks'): return [(_charset._encode(string, str(charset)), str(charset)) for string, charset in header._chunks] # If no encoding, just return the header with no charset. if not ecre.search(header): return [(header, None)] # First step is to parse all the encoded parts into triplets of the form # (encoded_string, encoding, charset). For unencoded strings, the last # two parts will be None. words = [] for line in header.splitlines(): parts = ecre.split(line) first = True while parts: unencoded = parts.pop(0) if first: unencoded = unencoded.lstrip() first = False if unencoded: words.append((unencoded, None, None)) if parts: charset = parts.pop(0).lower() encoding = parts.pop(0).lower() encoded = parts.pop(0) words.append((encoded, encoding, charset)) # Now loop over words and remove words that consist of whitespace # between two encoded strings. droplist = [] for n, w in enumerate(words): if n > 1 and w[1] and words[n-2][1] and words[n-1][0].isspace(): droplist.append(n-1) for d in reversed(droplist): del words[d] # The next step is to decode each encoded word by applying the reverse # base64 or quopri transformation. decoded_words is now a list of the # form (decoded_word, charset). decoded_words = [] for encoded_string, encoding, charset in words: if encoding is None: # This is an unencoded word. decoded_words.append((encoded_string, charset)) elif encoding == 'q': word = email.quoprimime.header_decode(encoded_string) decoded_words.append((word, charset)) elif encoding == 'b': # Postel's law: add missing padding paderr = len(encoded_string) % 4 if paderr: encoded_string += '==='[:4 - paderr] try: word = email.base64mime.decode(encoded_string) except binascii.Error: raise HeaderParseError('Base64 decoding error') else: decoded_words.append((word, charset)) else: raise AssertionError('Unexpected encoding: ' + encoding) # Now convert all words to bytes and collapse consecutive runs of # similarly encoded words. collapsed = [] last_word = last_charset = None for word, charset in decoded_words: if isinstance(word, unicode): word = bytes(word, 'raw-unicode-escape') if last_word is None: last_word = word last_charset = charset elif charset != last_charset: collapsed.append((last_word, last_charset)) last_word = word last_charset = charset elif last_charset is None: last_word += b' ' + word else: last_word += word collapsed.append((last_word, last_charset)) return collapsed if __name__ == "__main__": import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/headerprint.py ================================================ # vim: set fileencoding=utf-8 : # from __future__ import print_function import re from mailpile.util import md5_hex MUA_ML_HEADERS = (# Mailing lists are sending MUAs in their own right 'list-id', 'list-subscribe', 'list-unsubscribe') MUA_HP_HEADERS = ('date', 'from', 'to', 'reply-to', # We omit the Subject, because for some reason it seems # to jump around a lot. Same for CC. 'message-id', 'return-path', 'precedence', 'organization', 'mime-version', 'content-type', 'user-agent', 'x-mailer', 'x-mimeole', 'x-msmail-priority', 'x-priority', 'x-originating-ip', 'x-message-info', 'openpgp', 'x-openpgp', # Common services 'x-github-recipient', 'feedback-id', 'x-facebook') MUA_ID_HEADERS = ('x-mailer', 'user-agent', 'x-mimeole') HP_MUA_ID_SPACE = re.compile(r'(\s+)') HP_MUA_ID_IGNORE = re.compile(r'(\[[a-fA-F0-9%:]+\]|<\S+@\S+>' '|(mail|in)-[^\.]+|\d+)') HP_MUA_ID_SPLIT = re.compile(r'[\s,/;=()]+') HP_RECVD_PARSE = re.compile(r'(by\s+)' '[a-z0-9_\.-]*?([a-z0-9_-]*?\.?[a-z0-9_-]+\s+.*' 'with\s+.*)\s+id\s+.*$', flags=(re.MULTILINE + re.DOTALL)) def HeaderPrintMTADetails(message): """Extract details about the sender's outgoing SMTP server.""" details = [] # We want the first "non-local" received line. This can of course be # trivially spoofed, but looking at this will still protect against # all but the most targeted of spear phishing attacks. for rcvd in reversed(message.get_all('received') or []): if ('local' not in rcvd and ' mapi id ' not in rcvd and '127.0.0' not in rcvd and '[::1]' not in rcvd): parsed = HP_RECVD_PARSE.search(rcvd) if parsed: by = parsed.group(1) + parsed.group(2) by = HP_MUA_ID_SPACE.sub(' ', HP_MUA_ID_IGNORE.sub('x', by)) details = ['Received ' + by] break for h in ('DKIM-Signature', 'X-Google-DKIM-Signature'): for dkim in (message.get_all(h) or []): attrs = [HP_MUA_ID_SPACE.sub('', a) for a in dkim.split(';') if a.strip()[:1] in 'vacd'] details.extend([h, '; '.join(sorted(attrs))]) return details def HeaderPrintMUADetails(message, mta=None): """Summarize what the message tells us directly about the MUA.""" details = [] for header in MUA_ID_HEADERS: value = message.get(header) if value: # We want some details about the MUA, but also some stability. # Thus the HP_MUA_ID_IGNORE regexp... value = ' '.join([v for v in HP_MUA_ID_SPLIT.split(value.strip()) if not HP_MUA_ID_IGNORE.search(v)]) details.extend([header, value.strip()]) if not details: # FIXME: We could definitely make more educated guesses! if mta and mta[0].startswith('Received by google.com'): details.extend(['Guessed', 'GMail']) elif ('x-ms-tnef-correlator' in message or 'x-ms-has-attach' in message): details.extend(['Guessed', 'Exchange']) elif '@mailpile' in message.get('message-id', ''): details.extend(['Guessed', 'Mailpile']) return details def HeaderPrintGenericDetails(message, which=MUA_HP_HEADERS): """Extract message details which may help identify the MUA.""" return [k for k, v in message.items() if k.lower() in which] def HeaderPrints(message): """Generate fingerprints from message headers which identifies the MUA.""" m = HeaderPrintMTADetails(message) u = HeaderPrintMUADetails(message, mta=m)[:20] g = HeaderPrintGenericDetails(message)[:50] mua = (u[1] if u else None) if mua and mua.startswith('Mozilla '): mua = mua.split()[-1] return { # The sender-ID headerprints includes MTA info 'sender': md5_hex('\n'.join(m+u+g)), # Tool-chain headerprints ignore the MTA details 'tools': md5_hex('\n'.join(u+g)), # Our best guess about what the MUA actually is; may be None 'mua': mua} if __name__ == "__main__": import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/html.py ================================================ # vim: set fileencoding=utf-8 : # from __future__ import print_function import lxml.etree import lxml.html import lxml.html.clean import re RE_HTML_BORING = re.compile( '(\s+|(]*>\s*)+.*?)', flags=re.DOTALL|re.IGNORECASE) RE_EXCESS_WHITESPACE = re.compile( '\n\s*\n\s*', flags=re.DOTALL) RE_HTML_NEWLINES = re.compile( '(]*href=[\'"]?([^\'">]+)[^>]*>([^<]*)', flags=re.DOTALL|re.IGNORECASE) RE_HTML_IMGS = re.compile( ']*src=[\'"]?([^\'">]+)[^>]*>', flags=re.DOTALL|re.IGNORECASE) RE_HTML_IMG_ALT = re.compile( ']*alt=[\'"]?([^\'">]+)[^>]*>', flags=re.DOTALL|re.IGNORECASE) RE_XML_ENCODING = re.compile( '(<\?xml version=[^ ?>]*((?! +encoding=) [^ ?>]*)*)( +encoding=[^ ?>]*)', flags=re.DOTALL|re.IGNORECASE) # FIXME: Decide if this is strict enough or too strict...? SHARED_HTML_CLEANER = lxml.html.clean.Cleaner( page_structure=True, meta=True, links=True, javascript=True, scripts=True, frames=True, embedded=True, safe_attrs_only=True) def clean_html(html): # Find and delete possibly conflicting xml encoding # declaration to prevent lxml ValueError. # e.g. html = re.sub(RE_XML_ENCODING, r'\1', html).strip() return (SHARED_HTML_CLEANER.clean_html(html) if html else '') def extract_text_from_html(html, url_callback=None): try: # We compensate for some of the limitations of lxml... links, imgs = [], [] def delink(m): url, txt = m.group(1), m.group(2).strip() if url_callback is not None: url_callback(url, txt) if txt[:4] in ('http', 'www.'): return txt elif url.startswith('mailto:'): if '@' in txt: return txt else: return '%s (%s)' % (txt, url.split(':', 1)[1]) else: links.append(' [%d] %s%s' % (len(links) + 1, txt and (txt + ': ') or '', url)) return '%s[%d]' % (txt, len(links)) def deimg(m): tag, url = m.group(0), m.group(1) if ' alt=' in tag: return re.sub(RE_HTML_IMG_ALT, '\1', tag).strip() else: imgs.append(' [%d] %s' % (len(imgs)+1, url)) return '[Image %d]' % len(imgs) html = ( re.sub(RE_XML_ENCODING, r'\1', re.sub(RE_HTML_PARAGRAPHS, '\n\n\\1', re.sub(RE_HTML_NEWLINES, '\n\\1', re.sub(RE_HTML_BORING, ' ', re.sub(RE_HTML_LINKS, delink, re.sub(RE_HTML_IMGS, deimg, html ))))))).strip() if html: try: html_text = lxml.html.fromstring(html).text_content() except lxml.etree.Error: html_text = _('(Invalid HTML suppressed)') else: html_text = '' text = (html_text + (links and '\n\nLinks:\n' or '') + '\n'.join(links) + (imgs and '\n\nImages:\n' or '') + '\n'.join(imgs)) return re.sub(RE_EXCESS_WHITESPACE, '\n\n', text).strip() except: import traceback traceback.print_exc() return html if __name__ == "__main__": import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/safe.py ================================================ from __future__ import print_function import email import email.errors import email.message import random import re import rfc822 import time from urllib import quote, unquote from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailutils.header import decode_header from mailpile.util import * def safe_decode_hdr(msg=None, name=None, hdr=None, charset=None): """ This method stubbornly tries to decode header data and convert to Pythonic unicode strings. The strings are guaranteed not to contain tab, newline or carriage return characters. If used with a message object, the header and the MIME charset will be inferred from the message headers. >>> msg = email.message.Message() >>> msg['content-type'] = 'text/plain; charset=utf-8' >>> msg['from'] = 'G\\xc3\\xadsli R \\xc3\\x93la ' >>> safe_decode_hdr(msg, 'from') u'G\\xedsli R \\xd3la ' The =?...?= MIME header encoding is also recognized and processed. >>> safe_decode_hdr(hdr='=?iso-8859-1?Q?G=EDsli_R_=D3la?=\\r\\n') u'G\\xedsli R \\xd3la ' >>> safe_decode_hdr(hdr='"=?utf-8?Q?G=EDsli_R?= =?iso-8859-1?Q?=D3la?="') u'G\\xedsli R \\xd3la' And finally, guesses are made with raw binary data. This process could be improved, it currently only attempts utf-8 and iso-8859-1. >>> safe_decode_hdr(hdr='"G\\xedsli R \\xd3la"\\r\\t') u'"G\\xedsli R \\xd3la" ' >>> safe_decode_hdr(hdr='"G\\xc3\\xadsli R \\xc3\\x93la"\\n ') u'"G\\xedsli R \\xd3la" ' # See https://bugs.python.org/issue1079 # encoded word enclosed in parenthesis (comment syntax) >>> safe_decode_hdr(hdr='rene@example.com (=?utf-8?Q?Ren=C3=A9?=)') u'rene@example.com ( Ren\\xe9 )' # no space after encoded word >>> safe_decode_hdr(hdr='=?UTF-8?Q?Direction?=') u'Direction ' """ if hdr is None: value = msg and msg[name] or '' charset = charset or msg.get_content_charset() or 'utf-8' else: value = hdr charset = charset or 'utf-8' if not isinstance(value, unicode): # Already a str! Oh shit, might be nasty binary data. value = try_decode(value, charset, replace='?') # At this point we know we have a unicode string. Next we try # to very stubbornly decode and discover character sets. if '=?' in value and '?=' in value: try: # decode_header wants an unquoted str (not unicode) value = value.encode('utf-8').replace('"', '') # Decode! pairs = decode_header(value) value = ' '.join([try_decode(t, cs or charset) for t, cs in pairs]) except email.errors.HeaderParseError: pass # Finally, return the unicode data, with white-space normalized return value.replace('\r', ' ').replace('\t', ' ').replace('\n', ' ') def safe_parse_date(date_hdr): """Parse a Date: or Received: header into a unix timestamp.""" try: if ';' in date_hdr: date_hdr = date_hdr.split(';')[-1].strip() msg_ts = long(rfc822.mktime_tz(rfc822.parsedate_tz(date_hdr))) if (msg_ts > (time.time() + 24 * 3600)) or (msg_ts < 1): return None else: return msg_ts except (ValueError, TypeError, OverflowError): return None def safe_message_ts(msg, default=None, msg_mid=None, msg_id=None, session=None): """Extract a date, sanity checking against the Received: headers.""" hdrs = [safe_decode_hdr(msg, 'date')] + (msg.get_all('received') or []) dates = [safe_parse_date(date_hdr) for date_hdr in hdrs] msg_ts = dates[0] nz_dates = sorted([d for d in dates if d]) if nz_dates: a_week = 7 * 24 * 3600 # Ideally, we compare with the date on the 2nd SMTP relay, as # the first will often be the same host as composed the mail # itself. If we don't have enough hops, just use the last one. # # We don't want to use a median or average, because if the # message bounces around lots of relays or gets resent, we # want to ignore the latter additions. # rcv_ts = nz_dates[min(len(nz_dates)-1, 2)] # Now, if everything is normal, the msg_ts will be at nz_dates[0] # and it won't be too far away from our reference date. if (msg_ts == nz_dates[0]) and (abs(msg_ts - rcv_ts) < a_week): # Note: Trivially true for len(nz_dates) in (1, 2) return msg_ts # Damn, dates are screwy! # # Maybe one of the SMTP servers has a wrong clock? If the Date: # header falls within the range of all detected dates (plus a # week towards the past), still trust it. elif ((msg_ts >= (nz_dates[0]-a_week)) and (msg_ts <= nz_dates[-1])): return msg_ts # OK, Date: is insane, use one of the early Received: lines # instead. We picked the 2nd one above, that should do. else: if session and msg_mid and msg_id: session.ui.warning(_('=%s/%s using Received: instead of Date:' ) % (msg_mid, msg_id)) return rcv_ts else: # If the above fails, we assume the messages in the mailbox are in # chronological order and just add 1 second to the date of the last # message if date parsing fails for some reason. if session and msg_mid and msg_id: session.ui.warning(_('=%s/%s has a bogus date' ) % (msg_mid, msg_id)) return default def safe_get_msg_id(msg): raw_msg_id = safe_decode_hdr(msg, 'message-id') if not raw_msg_id: # Create a very long pseudo-msgid for messages without a # Message-ID. This was a very badly behaved mailer, so if # we create duplicates this way, we are probably only # losing spam. Even then the Received line should save us. raw_msg_id = ('\t'.join([safe_decode_hdr(msg, 'date'), safe_decode_hdr(msg, 'subject'), safe_decode_hdr(msg, 'received'), safe_decode_hdr(msg, 'from'), safe_decode_hdr(msg, 'to')]) # This is to avoid truncation in encode_msg_id: ).replace('<', '').strip() return raw_msg_id if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/mailutils/vcal.py ================================================ from __future__ import print_function import time import icalendar from datetime import datetime def calendar_parse(payload): c = icalendar.parser.Contentlines() lines = c.from_ical(payload) root = None obj = None for line in lines: if line == "": break parts = line.parts() if parts[0] == "BEGIN": t = vmap[parts[2]]() if not root: root = t if obj: obj.children.append(t) t.parent = obj obj = t root.stack.append(parts[2]) continue if parts[0] == "END": obj = obj.parent root.stack.pop() continue obj.add_part(*parts) return root.to_json() class VObject: def __init__(self): self.children = [] self.parent = None self.stack = [] self.parts = [] def add_part(self, key, params, value): self.parts.append([key, params, value]) def find_parts(self, key): res = [] for p in self.parts: if p[0] == key: res.append(p) return res def find_one_part(self, key): res = self.find_parts(key) if len(res) == 0: return None r = {"value": res[0][2], "params": res[0][1] } return r def find_one_part_value(self, key, value=None): res = self.find_parts(key) if len(res) == 0: return value return res[0][2] def get_datetime(self, key): val = self.find_one_part_value(key) try: return datetime(*time.strptime(val, "%Y%m%dT%H%M%SZ")[:6]) except: return datetime(*time.strptime(val, "%Y%m%dT%H%M%S")[:6]) def to_raw_json(self): parts = {} for p in self.parts: if p[0] not in parts: parts[p[0]] = [] parts[p[0]].append({"value": p[2], "parameters": p[1]}) children = [x.to_raw_json() for x in self.children] return { "type": self.__class__.__name__, "children": children, "parts": parts, } def to_json(self): return to_raw_json() class VTimeZone(VObject): pass class VTZStandard(VObject): pass class VTZDaylight(VObject): pass class VAlarm(VObject): pass class VEvent(VObject): def __init__(self): VObject.__init__(self) def to_json(self): summary = self.find_one_part_value("SUMMARY", "") description = self.find_one_part_value("DESCRIPTION", "").replace("\\n", "\n").replace("\n\n", "\n") dtstart = self.get_datetime("DTSTART") dtend = self.get_datetime("DTEND") location = self.find_one_part_value("LOCATION", "") attendees = [{"cn": x[1]["cn"], "email": x[2].split(":")[1]} for x in self.find_parts("ATTENDEE")] o = self.find_one_part("ORGANIZER") organizer = { "cn": o["params"]["CN"], "email": o["value"].split(":")[1]} tzinfo = None return { "summary": summary, "description": description, "dtstart": dtstart, "dtend": dtend, "location": location, "timezone": tzinfo, "organizer": organizer, "attendees": attendees, "alarms": [], } class VCalendar(VObject): def __init__(self): VObject.__init__(self) def print_events(self): for e in self.children: if isinstance(e, VEvent): print("%s invited you to %s" % (e.find_parts("ORGANIZER")[0][1]['CN'], e.find_parts("SUMMARY")[0][2])) print("%s" % e.find_parts("DTSTART")[0][2]) print("%s" % e.find_parts("LOCATION")[0][2]) def to_json(self): events = [] for e in self.children: # We are assuming VEvents will only occur immediately under the # VCalendar level. Haven't seen anything else in the wild. if isinstance(e, VEvent): events.append(e.to_json()) return events vmap = { "VALARM": VAlarm, "VTIMEZONE": VTimeZone, "VEVENT": VEvent, "VCALENDAR": VCalendar, "STANDARD": VTZStandard, "DAYLIGHT": VTZDaylight, } if __name__ == "__main__": cal = calendar_parse(open("calitem.cal").read()) # cal.print_tree() print("------------------------------") cal.print_events() print("------------------------------") ================================================ FILE: mailpile/packing.py ================================================ from __future__ import print_function import struct import time import zlib from mailpile.util import * def PackIntSet(ints): """ Pack a set of ints to a compact string, unpackable by UnpackIntSet. Short lists are binary packed directly, but long lists are converted to a bitmask and then compressed using zlib. >>> intset = set([1, 5, 9, 10000]) >>> intsetstr = PackIntSet(intset) >>> type(intsetstr), len(intsetstr) (, 16) >>> UnpackIntSet(intsetstr) == intset True >>> intset = set(list(range(1000, 50000) + [1, 2, 3])) >>> intsetstr = PackIntSet(intset) >>> intsetstr.startswith('\xff\xff\xff\xff') True >>> len(intsetstr) 37 >>> UnpackIntSet(intsetstr) == intset True """ if len(ints) > 15: return '\xff\xff\xff\xff' + zlib.compress(intlist_to_bitmask(ints)) else: return struct.pack('<' + 'I' * len(ints), *ints) def UnpackIntSet(data): """ Unpack a set of ints previously packed using PackIntSet. """ if len(data) > 13 and data[:4] == '\xff\xff\xff\xff': return set(bitmask_to_intlist(zlib.decompress(data[4:]))) else: return set(struct.unpack('<' + 'I' * (len(data)//4), data)) def PackLongList(longs): """ Pack a list of longs to a compact string, unpackable by UnpackLongList. Short lists are binary packed directly: >>> ll = [1, 5, 100000000000L] >>> llstr = PackLongList(ll) >>> UnpackLongList(llstr) == ll True >>> type(llstr), len(llstr) (, 24) Longer lists are zlib compressed, which can result in significant space savings for many types of data. >>> ll += list(range(100, 1000)) >>> llstr = PackLongList(ll) >>> llstr.startswith('\xff\xff\xff\xff\xff\xff\xff\xff') True >>> UnpackLongList(llstr) == ll True >>> len(llstr) 1416 """ packed = struct.pack('<' + 'q' * len(longs), *longs) if (len(packed) > 8 * 15) or (longs[0] == 0xffffffffffffffff): return ('\xff\xff\xff\xff\xff\xff\xff\xff' + zlib.compress(packed)) else: return packed def UnpackLongList(data): """ Unpack a list of longs previously packed using PackLongList. """ if len(data) > 17 and data[:8] == '\xff\xff\xff\xff\xff\xff\xff\xff': data = zlib.decompress(data[8:]) return list(struct.unpack('<' + 'q' * (len(data)//8), data)) class StorageBackedData(object): """ This lovely hack exposes the full API of a Python set or list, but any writes get flushed to a storage backend and the initial state is loaded from the same. It is NOT SAFE to ever have more than one of these for a given backend as they will not stay in sync. Since most methods are proxies, using set method on a backed list will fail and vice-versa. This class must be subclassed and _pack and _unpack implemented. """ def __init__(self, storage, skey): self._storage = storage self._skey = skey self.load() self.last_save = time.time() self.auto_save = True self.interval = -1 self.dirty = False def _pack(self, data): raise NotImplemented() def _unpack(self, data): raise NotImplemented() def load(self): try: self._obj = self._unpack(self._storage[self._skey]) except (KeyError, IndexError): self._obj = self._unpack('') def save(self, maybe=False): if not maybe or self.dirty: self._storage[self._skey] = self._pack(self._obj) self.dirty = False def _dirty_maybe_save(self): self.dirty = True if self.auto_save: if (self.interval < 1 or self.last_save < time.time() - self.interval): self.save() def _r(self, method, *args, **kwargs): return getattr(self._obj, method)(*args, **kwargs) def _w(self, method, *args, **kwargs): rv = getattr(self._obj, method)(*args, **kwargs) self._dirty_maybe_save() return rv def _iw(self, method, *args, **kwargs): self._obj = getattr(self._obj, method)(*args, **kwargs) self._dirty_maybe_save() return self def __and__(s, *a, **kw): return s._r('__and__', *a, **kw) def __cmp__(s, *a, **kw): return s._r('__cmp__', *a, **kw) def __contains__(s, *a, **kw): return s._r('__contains__', *a, **kw) def __eq__(s, *a, **kw): return s._r('__eq__', *a, **kw) def __ge__(s, *a, **kw): return s._r('__ge__', *a, **kw) def __getitem__(s, *a, **kw): return s._r('__getitem__', *a, **kw) def __getslice__(s, *a, **kw): return s._r('__getslice__', *a, **kw) def __gt__(s, *a, **kw): return s._r('__gt__', *a, **kw) def __iter__(s, *a, **kw): return s._r('__iter__', *a, **kw) def __le__(s, *a, **kw): return s._r('__le__', *a, **kw) def __len__(s, *a, **kw): return s._r('__len__', *a, **kw) def __lt__(s, *a, **kw): return s._r('__lt__', *a, **kw) def __mul__(s, *a, **kw): return s._r('__mul__', *a, **kw) def __ne__(s, *a, **kw): return s._r('__ne__', *a, **kw) def __or__(s, *a, **kw): return s._r('__or__', *a, **kw) def __rand__(s, *a, **kw): return s._r('__rand__', *a, **kw) def __reduce__(s, *a, **kw): return s._r('__reduce__', *a, **kw) def __repr__(s, *a, **kw): return s._r('__repr__', *a, **kw) def __reversed__(s, *a, **kw): return s._r('__reversed__', *a, **kw) def __rmul__(s, *a, **kw): return s._r('__rmul__', *a, **kw) def __rsub__(s, *a, **kw): return s._r('__rsub__', *a, **kw) def __rxor__(s, *a, **kw): return s._r('__rxor__', *a, **kw) def __sizeof__(s, *a, **kw): return s._r('__sizeof__', *a, **kw) def __sub__(s, *a, **kw): return s._r('__sub__', *a, **kw) def __xor__(s, *a, **kw): return s._r('__xor__', *a, **kw) def copy(s, *a, **kw): return s._r('copy', *a, **kw) def count(s, *a, **kw): return s._r('count', *a, **kw) def difference(s, *a, **kw): return s._r('difference', *a, **kw) def index(s, *a, **kw): return s._r('index', *a, **kw) def intersection(s, *a, **kw): return s._r('intersection', *a, **kw) def isdisjoint(s, *a, **kw): return s._r('isdisjoint', *a, **kw) def issubset(s, *a, **kw): return s._r('issubset', *a, **kw) def issuperset(s, *a, **kw): return s._r('issuperset', *a, **kw) def union(s, *a, **kw): return s._r('union', *a, **kw) def symmetric_difference(s, *a, **kw): return s._r('symmetric_difference', *a, **kw) def __iadd__(s, *a, **kw): return s._iw('__iadd__', *a, **kw) def __iand__(s, *a, **kw): return s._iw('__iand__', *a, **kw) def __imul__(s, *a, **kw): return s._iw('__imul__', *a, **kw) def __ior__(s, *a, **kw): return s._iw('__ior__', *a, **kw) def __isub__(s, *a, **kw): return s._iw('__isub__', *a, **kw) def __ixor__(s, *a, **kw): return s._iw('__ixor__', *a, **kw) def __delitem__(s, *a, **kw): return s._w('__delitem__', *a, **kw) def __delslice__(s, *a, **kw): return s._w('__delslice__', *a, **kw) def __setitem__(s, *a, **kw): return s._w('__setitem__', *a, **kw) def __setslice__(s, *a, **kw): return s._w('__setslice__', *a, **kw) def add(s, *a, **kw): return s._w('add', *a, **kw) def append(s, *a, **kw): return s._w('append', *a, **kw) def clear(s, *a, **kw): return s._w('clear', *a, **kw) def discard(s, *a, **kw): return s._w('discard', *a, **kw) def extend(s, *a, **kw): return s._w('extend', *a, **kw) def insert(s, *a, **kw): return s._w('insert', *a, **kw) def pop(s, *a, **kw): return s._w('pop', *a, **kw) def remove(s, *a, **kw): return s._w('remove', *a, **kw) def reverse(s, *a, **kw): return s._w('reverse', *a, **kw) def sort(s, *a, **kw): return s._w('sort', *a, **kw) def update(s, *a, **kw): return s._w('update', *a, **kw) def difference_update(s, *a, **kw): return s._w('difference_update', *a, **kw) def intersection_update(s, *a, **kw): return s._w('intersection_update', *a, **kw) def symmetric_difference_update(s, *a, **kw): return s._w('symmetric_difference_update', *a, **kw) class StorageBackedSet(StorageBackedData): """ This combines StorageBackedData with Pack/UnpackIntSet to pack and save sets of ints. >>> storage = {'sbs': '\\x01\\x00\\x00\\x00'} >>> sbs = StorageBackedSet(storage, 'sbs') >>> 1 in sbs True >>> sbs.add(2) >>> sbs.save() >>> UnpackIntSet(storage['sbs']) == set([1, 2]) True """ def _pack(self, data): return PackIntSet(data) def _unpack(self, data): return UnpackIntSet(data) class StorageBackedLongs(StorageBackedData): """ This combines StorageBackedData with Pack/UnpackLongList to pack and save sets of ints. >>> storage = {'sbl': '\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00'} >>> sbl = StorageBackedLongs(storage, 'sbl') >>> 1 in sbl True >>> sbl.append(2) >>> sbl.save() >>> UnpackLongList(storage['sbl']) == [1, 2] True """ def _pack(self, data): return PackLongList(data) def _unpack(self, data): return UnpackLongList(data) if __name__ == '__main__': import doctest import sys results = doctest.testmod(optionflags=doctest.ELLIPSIS, extraglobs={}) print('%s' % (results, )) if results.failed: sys.exit(1) ================================================ FILE: mailpile/platforms.py ================================================ """ This module tries to centralize most of the platform-specific code in use by Mailpile. If you find yourself checking which platform the app runs on, adding a function here instead is probably The Right Thing. """ import copy import os import subprocess import sys # This is a cache of discovered binaries and their paths. BINARIES = {} # These are the binaries we want, and the test we use to detect whether # they are available/working. # BINARIES_WANTED = {# Test command Required? 'GnuPG': (['gpg', '--version'], True), 'GnuPG_dirmngr': (['dirmngr', '--version'], True), 'GnuPG_agent': (['gpg-agent', '--version'], True), 'OpenSSL': (['openssl', 'version'], True), 'Tor': (['tor', '--version'], False)} def _assert_file_exists(path): if not os.path.exists(path): raise OSError('Not found: %s' % path) return path def DetectBinaries( which=None, use_cache=True, preferred={}, skip=None, _raise=None): import mailpile.util import mailpile.safe_popen import traceback global BINARIES if which and use_cache: if which in BINARIES: return BINARIES[which] env_bin = os.getenv('MAILPILE_%s' % which.upper(), '') if env_bin: BINARIES[which] = env_bin return env_bin if skip is None: skip = (os.getenv('MAILPILE_IGNORE_BINARIES', '') .replace('/ga', '_agent') # Backwards compatibility .replace('/dm', '_dirmngr') # Backwards compatibility .split()) def _run_bintest(bt): p = mailpile.safe_popen.Popen(bt, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return p.communicate() for binary, (bin_test, reqd) in BINARIES_WANTED.iteritems(): if binary in skip: continue if (which is None) or (binary == which): if preferred.get(binary): bin_test = copy.copy(bin_test) bin_test[0] = preferred[binary] else: env_bin = os.getenv('MAILPILE_%s' % binary.upper(), '') if env_bin: BINARIES[binary] = env_bin continue try: mailpile.util.RunTimed(5.0, _run_bintest, bin_test) BINARIES[binary] = bin_test[0] if (not os.path.dirname(BINARIES[binary]) and not sys.platform.startswith('win')): try: path = subprocess.check_output(['which', BINARIES[binary]]) if path: BINARIES[binary] = path.strip() except (OSError, subprocess.CalledProcessError): pass except (OSError, subprocess.CalledProcessError, mailpile.util.TimedOut): if binary in BINARIES: del BINARIES[binary] if which: if _raise not in (None, False): if not BINARIES.get(which): raise _raise('%s not found' % which) return BINARIES.get(which) elif _raise not in (None, False): for binary, (bin_test, reqd) in BINARIES_WANTED.iteritems(): if binary in skip or not reqd: continue if not BINARIES.get(binary): raise _raise('%s not found' % binary) return BINARIES def GetDefaultGnuPGCommand(_raise=OSError): return DetectBinaries(which='GnuPG', _raise=_raise) def GetDefaultOpenSSLCommand(_raise=OSError): return DetectBinaries(which='OpenSSL', _raise=_raise) def GetDefaultTorPath(_raise=OSError): return DetectBinaries(which='Tor', _raise=_raise) def InDesktopEnvironment(): """ Returns True if we're running in a desktop environment of some sort. """ # FIXME: Detect if we are somehow in the background on Windows or OS X. return (sys.platform[:3] in ('dar', 'win') or os.getenv('DISPLAY')) def RenameCannotOverwrite(): """ The os.rename() function will not overwrite existing files on Windows. """ return sys.platform.startswith('win') def NeedExplicitPortCheck(): """ Our HTTP worker doesn't detect port reuse on Windows, need explicit checks. """ return sys.platform.startswith('win') def TerminalSupportsAnsiColors(): """ Windows doesn't like ANSI colors. Also, we want a TTY. """ return (sys.stdout.isatty() and sys.platform[:3] != "win") def WindowsPopenSemantics(): """ The safe_popen module implements slightly different semantics on Windows. """ return sys.platform.startswith('win') def GetAppDataDirectory(): if sys.platform.startswith('win'): # Obey Windows conventions (more or less?) return os.getenv('APPDATA', os.path.expanduser('~')) elif sys.platform.startswith('darwin'): # Obey Mac OS X conventions return os.path.expanduser('~/Library/Application Support') else: # Assume other platforms are Unixy return os.getenv('XDG_DATA_HOME', os.path.expanduser('~/.local/share')) def RestrictReadAccess(path): """ Restrict access to a file or directory so only the user can read it. """ # FIXME: Windows code goes here! if os.path.isdir(path): os.chmod(path, 0o700) else: os.chmod(path, 0o600) def RandomListeningPort(count=1, host='127.0.0.1'): socks = [] ports = [] try: import socket for port in range(0, count): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((host, 0)) socks.append(sock) ports.append(sock.getsockname()[1]) if count == 1: return ports[0] else: return ports finally: for sock in socks: sock.close() ================================================ FILE: mailpile/plugins/__init__.py ================================================ from __future__ import print_function # Plugins! import imp import inspect import json import os import sys import traceback import mailpile.commands import mailpile.config.defaults import mailpile.vcard from mailpile.i18n import i18n_disabled from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import register as register_mailbox from mailpile.util import * ##[ Plugin discovery ]######################################################## # These are the plugins we ship/import by default __all__ = [ 'core', 'eventlog', 'search', 'tags', 'contacts', 'compose', 'groups', 'dates', 'sizes', 'autotag', 'cryptostate', 'crypto_gnupg', 'gui', 'setup_magic', 'oauth', 'exporters', 'plugins', 'motd', 'backups', 'vcard_carddav', 'vcard_gnupg', 'vcard_gravatar', 'vcard_libravatar', 'vcard_mork', 'html_magic', 'migrate', 'smtp_server', 'crypto_policy', 'keylookup', 'webterminal', 'crypto_autocrypt' ] PLUGINS = __all__ class EmailTransform(object): """Base class for e-mail transforms""" def __init__(self, config): self.config = config def _get_sender_profile(self, sender, kwargs): profile = kwargs.get('sender_profile') if not profile: profile = self.config.get_profile(sender) return profile def _get_first_part(self, msg, mimetype): for part in msg.walk(): if not part.is_multipart(): mimetype = (part.get_content_type() or 'text/plain').lower() if mimetype == 'text/plain': return part return None def TransformIncoming(self, *args, **kwargs): return list(args[:]) + [False] def TransformOutgoing(self, *args, **kwargs): return list(args[:]) + [False, True] class PluginError(Exception): pass class PluginManager(object): """ Manage importing and loading of plugins. Note that this class is effectively a singleton, as it works entirely with globals within the mailpile.plugins module. """ DEFAULT = __all__ BUILTIN = (DEFAULT + [ 'autotag_sb' ]) # These are plugins which we consider required REQUIRED = [ 'core', 'eventlog', 'search', 'tags', 'contacts', 'compose', 'groups', 'dates', 'sizes', 'cryptostate', 'setup_magic', 'oauth', 'html_magic', 'plugins', 'keylookup', 'motd', 'backups', 'gui' ] # Plugins we want, if they are discovered WANTED = [ 'autoajax', 'print', 'hints' ] # Plugins that have been renamed from past releases RENAMED = { 'crypto_utils': 'crypto_gnupg' } DISCOVERED = {} LOADED = [] def __init__(self, plugin_name=None, builtin=False, deprecated=False, config=None, session=None): if builtin and isinstance(builtin, (str, unicode)): builtin = os.path.basename(builtin) for ignore in ('.py', '.pyo', '.pyc'): if builtin.endswith(ignore): builtin = builtin[:-len(ignore)] if builtin not in self.LOADED: self.LOADED.append(builtin) self.loading_plugin = plugin_name self.loading_builtin = plugin_name and builtin self.builtin = builtin self.deprecated = deprecated self.session = session self.config = config self.manifests = [] def _listdir(self, path): try: return [d for d in os.listdir(path) if not d.startswith('.')] except OSError: return [] def _uncomment(self, json_data): return '\n'.join([l for l in json_data.splitlines() if not l.strip().startswith('#')]) def discover(self, paths, update=False): """ Scan the plugin directories for plugins we could load. This updates the global PluginManager state and returns the PluginManager itself (for chaining). """ plugins = self.BUILTIN[:] for pdir in paths: for subdir in self._listdir(pdir): pname = subdir.lower() if pname in self.BUILTIN: print('Cannot overwrite built-in plugin: %s' % pname) continue if pname in self.DISCOVERED and not update: # FIXME: this is lame # print 'Ignoring duplicate plugin: %s' % pname continue plug_path = os.path.join(pdir, subdir) manifest_filename = os.path.join(plug_path, 'manifest.json') try: with open(manifest_filename) as mfd: manifest = json.loads(self._uncomment(mfd.read())) safe_assert(manifest.get('name') == subdir) # FIXME: Need more sanity checks self.DISCOVERED[pname] = (plug_path, manifest) except (ValueError, AssertionError): print('Bad manifest: %s' % manifest_filename) except (OSError, IOError): pass return self def available(self): return self.BUILTIN[:] + self.DISCOVERED.keys() def loadable(self): return self.BUILTIN[:] + self.RENAMED.keys() + self.DISCOVERED.keys() def loadable_early(self): return [k for k, (n, m) in self.DISCOVERED.iteritems() if not m.get('require_login', True)] def _import(self, full_name, full_path): # create parents as necessary parents = full_name.split('.')[2:] # skip mailpile.plugins module = "mailpile.plugins" for parent in parents: mp = '%s.%s' % (module, parent) if mp not in sys.modules: sys.modules[mp] = imp.new_module(mp) sys.modules[module].__dict__[parent] = sys.modules[mp] module = mp safe_assert(module == full_name) # load actual module sys.modules[full_name].__file__ = full_path with i18n_disabled: with open(full_path, 'r') as mfd: exec(mfd.read(), sys.modules[full_name].__dict__) def _load(self, plugin_name, process_manifest=False, config=None): full_name = 'mailpile.plugins.%s' % plugin_name if full_name in sys.modules: return self self.loading_plugin = full_name if plugin_name in self.BUILTIN: # The builtins are just normal Python code. If they have a # manifest, they'll invoke process_manifest themselves. self.loading_builtin = True module = __import__(full_name) elif plugin_name in self.DISCOVERED: dirname, manifest = self.DISCOVERED[plugin_name] self.loading_builtin = False # Load the Python requested by the manifest.json files = manifest.get('code', {}).get('python', []) try: for filename in files: path = os.path.join(dirname, filename) if filename == '.': self._import(full_name, dirname) continue elif filename.endswith('.py'): subname = filename[:-3].replace('/', '.') # FIXME: Is this a good idea? if full_name.endswith('.'+subname): self._import(full_name, path) continue elif os.path.isdir(path): subname = filename.replace('/', '.') else: continue self._import('.'.join([full_name, subname]), path) except KeyboardInterrupt: raise except: traceback.print_exc(file=sys.stderr) print('FIXME: Loading %s failed, tell user!' % full_name) if full_name in sys.modules: del sys.modules[full_name] return None spec = (full_name, manifest, dirname) self.manifests.append(spec) if process_manifest: self._process_manifest_pass_one(*spec) self._process_manifest_pass_two(*spec) self._process_startup_hooks(*spec) else: print('Unrecognized plugin: %s' % plugin_name) return self if plugin_name not in self.LOADED: self.LOADED.append(plugin_name) return self def load(self, *args, **kwargs): try: return self._load(*args, **kwargs) finally: self.loading_plugin = None self.loading_builtin = False def process_shutdown_hooks(self): for plugin_name in self.DISCOVERED.keys(): try: package = 'mailpile.plugins.%s' % plugin_name _, manifest = self.DISCOVERED[plugin_name] if package in sys.modules: for method_name in self._mf_path(manifest, 'lifecycle', 'shutdown'): method = self._get_method(package, method_name) method(self.config) except: # ignore exceptions here as mailpile is going to shut down traceback.print_exc(file=sys.stderr) def process_manifests(self): failed = [] for process in (self._process_manifest_pass_one, self._process_manifest_pass_two, self._process_startup_hooks): for spec in self.manifests: try: if spec[0] not in failed: process(*spec) except Exception as e: print('Failed to process manifest for %s: %s' % (spec[0], e)) failed.append(spec[0]) traceback.print_exc() return self def _mf_path(self, mf, *path): for p in path: mf = mf.get(p, {}) return mf def _mf_iteritems(self, mf, *path): return self._mf_path(mf, *path).iteritems() def _get_method(self, full_name, method): full_method_name = '.'.join([full_name, method]) package, method_name = full_method_name.rsplit('.', 1) module = sys.modules[package] return getattr(module, method_name) def _get_class(self, full_name, class_name): full_class_name = '.'.join([full_name, class_name]) mod_name, class_name = full_class_name.rsplit('.', 1) module = __import__(mod_name, globals(), locals(), class_name) return getattr(module, class_name) def _process_manifest_pass_one(self, full_name, manifest=None, plugin_path=None): """ Pass one of processing the manifest data. This updates the global configuration and registers Python code with the URL map. """ if not manifest: return manifest_path = lambda *p: self._mf_path(manifest, *p) manifest_iteritems = lambda *p: self._mf_iteritems(manifest, *p) # Register config variables and sections for section, rules in manifest_iteritems('config', 'sections'): self.register_config_section(*(section.split('.') + [rules])) for section, rules in manifest_iteritems('config', 'variables'): self.register_config_variables(*(section.split('.') + [rules])) # Register commands for command in manifest_path('commands'): cls = self._get_class(full_name, command['class']) # FIXME: This is all a bit hacky, we probably just want to # kill the SYNOPSIS attribute entirely. if 'input' in command: name = url = '%s/%s' % (command['input'], command['name']) cls.UI_CONTEXT = command['input'] else: name = command.get('name', cls.SYNOPSIS[1]) url = command.get('url', cls.SYNOPSIS[2]) cls.SYNOPSIS = tuple([cls.SYNOPSIS[0], name, url, cls.SYNOPSIS_ARGS or cls.SYNOPSIS[3]]) self.register_commands(cls) # Register worker threads for thr in manifest_path('threads'): self.register_worker(self._get_class(full_name, thr)) # Register mailboxes package = str(full_name) for mailbox in manifest_path('mailboxes'): cls = self._get_class(package, mailbox['class']) priority = int(mailbox['priority']) register_mailbox(priority, cls) def _process_manifest_pass_two(self, full_name, manifest=None, plugin_path=None): """ Pass two of processing the manifest data. This maps templates and data to API commands and links registers classes and methods as hooks here and there. As these things depend both on configuration and the URL map, this happens as a second phase. """ if not manifest: return manifest_path = lambda *p: self._mf_path(manifest, *p) manifest_iteritems = lambda *p: self._mf_iteritems(manifest, *p) # Register javascript classes for fn in manifest.get('code', {}).get('javascript', []): class_name = fn.replace('/', '.').rsplit('.', 1)[0] # FIXME: Is this a good idea? if full_name.endswith('.'+class_name): parent, class_name = full_name.rsplit('.', 1) else: parent = full_name self.register_js(parent, class_name, os.path.join(plugin_path, fn)) # Register CSS files for fn in manifest.get('code', {}).get('css', []): file_name = fn.replace('/', '.').rsplit('.', 1)[0] self.register_css(full_name, file_name, os.path.join(plugin_path, fn)) # Register web assets if plugin_path: from mailpile.urlmap import UrlMap um = UrlMap(session=self.session, config=self.config) for url, info in manifest_iteritems('routes'): filename = os.path.join(plugin_path, info['file']) # Short-cut for static content if url.startswith('/static/'): self.register_web_asset(full_name, url[8:], filename, mimetype=info.get('mimetype', None)) continue # Finds the right command class and register asset in # the right place for that particular command. commands = [] if (not url.startswith('/api/')) and 'api' in info: url = '/api/%d%s' % (info['api'], url) if url[-1] == '/': url += 'as.html' for method in ('GET', 'POST', 'PUT', 'UPDATE', 'DELETE'): try: commands = um.map(None, method, url, {}, {}) break except UsageError: pass output = [o.get_render_mode() for o in commands if hasattr(o, 'get_render_mode')] output = output and output[-1] or 'html' if commands: command = commands[-1] tpath = command.template_path(output.split('.')[-1], template=output) self.register_web_asset(full_name, 'html/' + tpath, filename) else: print('FIXME: Un-routable URL in manifest %s' % url) # Register email content/crypto hooks s = self for which, reg in ( ('outgoing_content', s.register_outgoing_email_content_transform), ('outgoing_crypto', s.register_outgoing_email_crypto_transform), ('incoming_crypto', s.register_incoming_email_crypto_transform), ('incoming_content', s.register_incoming_email_content_transform) ): for item in manifest_path('email_transforms', which): name = '%3.3d_%s' % (int(item.get('priority', 999)), full_name) reg(name, self._get_class(full_name, item['class'])) # Register search keyword extractors s = self for which, reg in ( ('meta', s.register_meta_kw_extractor), ('text', s.register_text_kw_extractor), ('data', s.register_data_kw_extractor) ): for item in manifest_path('keyword_extractors', which): reg('%s.%s' % (full_name, item), self._get_class(full_name, item)) # Register contact/vcard hooks for which, reg in ( ('importers', self.register_vcard_importers), ('exporters', self.register_contact_exporters), ('context', self.register_contact_context_providers) ): for item in manifest_path('contacts', which): reg(self._get_class(full_name, item)) # Register periodic jobs def reg_job(info, spd, register): interval, cls = info['interval'], info['class'] callback = self._get_class(full_name, cls) register('%s.%s/%s-%s' % (full_name, cls, spd, interval), interval, callback) for info in manifest_path('periodic_jobs', 'fast'): reg_job(info, 'fast', self.register_fast_periodic_job) for info in manifest_path('periodic_jobs', 'slow'): reg_job(info, 'slow', self.register_slow_periodic_job) ucfull_name = full_name.capitalize() for ui_type, elems in manifest.get('user_interface', {}).iteritems(): for hook in elems: if 'javascript_setup' in hook: js = hook['javascript_setup'] if not js.startswith('Mailpile.'): hook['javascript_setup'] = '%s.%s' % (ucfull_name, js) if 'javascript_events' in hook: for event, call in hook['javascript_events'].iteritems(): if not call.startswith('Mailpile.'): hook['javascript_events'][event] = '%s.%s' \ % (ucfull_name, call) self.register_ui_element(ui_type, **hook) def _process_startup_hooks(self, package, manifest=None, plugin_path=None): if not manifest: return manifest_path = lambda *p: self._mf_path(manifest, *p) for method_name in manifest_path('lifecycle', 'startup'): method = self._get_method(package, method_name) method(self.config) def _compat_check(self, strict=True): if ((strict and (not self.loading_plugin and not self.builtin)) or self.deprecated): stack = inspect.stack() if str(stack[2][1]) == '': raise PluginError('Naughty plugin tried to directly access ' 'mailpile.plugins!') where = '->'.join(['%s:%s' % ('/'.join(stack[i][1].split('/')[-2:]), stack[i][2]) for i in reversed(range(2, len(stack)-1))]) print(('FIXME: Deprecated use of %s at %s (issue #547)' ) % (stack[1][3], where)) def _rhtf(self, kw_hash, term, function): if term in kw_hash: raise PluginError('Already registered: %s' % term) kw_hash[term] = function ##[ Pluggable configuration ]############################################# def register_config_variables(self, *args): self._compat_check() args = list(args) rules = args.pop(-1) dest = mailpile.config.defaults.CONFIG_RULES path = '/'.join(args) for arg in args: dest = dest[arg][-1] for rname, rule in rules.iteritems(): if rname in dest: raise PluginError('Variable already exist: %s/%s' % (path, rname)) else: dest[rname] = rule def register_config_section(self, *args): self._compat_check() args = list(args) rules = args.pop(-1) rname = args.pop(-1) dest = mailpile.config.defaults.CONFIG_RULES path = '/'.join(args) for arg in args: dest = dest[arg][-1] if rname in dest: raise PluginError('Section already exist: %s/%s' % (path, rname)) else: dest[rname] = rules ##[ Pluggable message transformations ]################################### INCOMING_EMAIL_ENCRYPTION = {} INCOMING_EMAIL_CONTENT = {} OUTGOING_EMAIL_CONTENT = {} OUTGOING_EMAIL_ENCRYPTION = {} def _txf_in(self, transforms, config, msg, kwargs): matched = 0 for name in sorted(transforms.keys()): txf = transforms[name](config) msg, match, cont = txf.TransformIncoming(msg, **kwargs) if match: matched += 1 if not cont: break return msg, matched def _txf_out(self, transforms, cfg, s, r, msg, kwa): matched = 0 for name in sorted(transforms.keys()): txf = transforms[name](cfg) s, r, msg, match, cont = txf.TransformOutgoing(s, r, msg, **kwa) if match: matched += 1 if not cont: break return s, r, msg, matched def incoming_email_crypto_transform(self, cfg, msg, **kwa): return self._txf_in(self.INCOMING_EMAIL_ENCRYPTION, cfg, msg, kwa) def incoming_email_content_transform(self, config, msg, **kwa): return self._txf_in(self.INCOMING_EMAIL_CONTENT, config, msg, kwa) def outgoing_email_content_transform(self, cfg, s, r, m, **kwa): return self._txf_out(self.OUTGOING_EMAIL_CONTENT, cfg, s, r, m, kwa) def outgoing_email_crypto_transform(self, cfg, s, r, m, **kwa): return self._txf_out(self.OUTGOING_EMAIL_ENCRYPTION, cfg, s, r, m, kwa) def register_incoming_email_crypto_transform(self, name, transform): return self._rhtf(self.INCOMING_EMAIL_ENCRYPTION, name, transform) def register_incoming_email_content_transform(self, name, transform): return self._rhtf(self.INCOMING_EMAIL_CONTENT, name, transform) def register_outgoing_email_content_transform(self, name, transform): return self._rhtf(self.OUTGOING_EMAIL_CONTENT, name, transform) def register_outgoing_email_crypto_transform(self, name, transform): return self._rhtf(self.OUTGOING_EMAIL_ENCRYPTION, name, transform) ##[ Pluggable keyword extractors ]######################################## DATA_KW_EXTRACTORS = {} TEXT_KW_EXTRACTORS = {} META_KW_EXTRACTORS = {} def register_data_kw_extractor(self, term, function): self._compat_check() return self._rhtf(self.DATA_KW_EXTRACTORS, term, function) def register_text_kw_extractor(self, term, function): self._compat_check() return self._rhtf(self.TEXT_KW_EXTRACTORS, term, function) def register_meta_kw_extractor(self, term, function): self._compat_check() return self._rhtf(self.META_KW_EXTRACTORS, term, function) def get_data_kw_extractors(self): self._compat_check(strict=False) return self.DATA_KW_EXTRACTORS.values() def get_text_kw_extractors(self): self._compat_check(strict=False) return self.TEXT_KW_EXTRACTORS.values() def get_meta_kw_extractors(self): self._compat_check(strict=False) return self.META_KW_EXTRACTORS.values() ##[ Pluggable search terms ]############################################## SEARCH_TERMS = {} def get_search_term(self, term, default=None): self._compat_check(strict=False) return self.SEARCH_TERMS.get(term, default) def register_search_term(self, term, function): self._compat_check() if term in self.SEARCH_TERMS: raise PluginError('Already registered: %s' % term) self.SEARCH_TERMS[term] = function ##[ Pluggable keyword filters ]########################################### FILTER_HOOKS_PRE = {} FILTER_HOOKS_POST = {} def get_filter_hooks(self, hooks): self._compat_check(strict=False) return ([self.FILTER_HOOKS_PRE[k] for k in sorted(self.FILTER_HOOKS_PRE.keys())] + hooks + [self.FILTER_HOOKS_POST[k] for k in sorted(self.FILTER_HOOKS_POST.keys())]) def register_filter_hook_pre(self, name, hook): self._compat_check() self.FILTER_HOOKS_PRE[name] = hook def register_filter_hook_post(self, name, hook): self._compat_check() self.FILTER_HOOKS_POST[name] = hook ##[ Pluggable vcard functions ]########################################### VCARD_IMPORTERS = {} VCARD_EXPORTERS = {} VCARD_CONTEXT_PROVIDERS = {} def _reg_vcard_plugin(self, what, cfg_sect, plugin_classes, cls, dct): for plugin_class in plugin_classes: if not plugin_class.SHORT_NAME or not plugin_class.FORMAT_NAME: raise PluginError("Please set SHORT_NAME " "and FORMAT_* attributes!") if not issubclass(plugin_class, cls): raise PluginError("%s must be a %s" % (what, cls)) if plugin_class.SHORT_NAME in dct: raise PluginError("%s for %s already registered" % (what, importer.FORMAT_NAME)) if plugin_class.CONFIG_RULES: rules = { 'guid': ['VCard source UID', str, ''], 'description': ['VCard source description', str, ''] } rules.update(plugin_class.CONFIG_RULES) self.register_config_section( 'prefs', 'vcard', cfg_sect, plugin_class.SHORT_NAME, [plugin_class.FORMAT_DESCRIPTION, rules, []]) dct[plugin_class.SHORT_NAME] = plugin_class def register_vcard_importers(self, *importers): self._compat_check() self._reg_vcard_plugin('Importer', 'importers', importers, mailpile.vcard.VCardImporter, self.VCARD_IMPORTERS) def register_contact_exporters(self, *exporters): self._compat_check() self._reg_vcard_plugin('Exporter', 'exporters', exporters, mailpile.vcard.VCardExporter, self.VCARD_EXPORTERS) def register_contact_context_providers(self, *providers): self._compat_check() self._reg_vcard_plugin('Context provider', 'context', providers, mailpile.vcard.VCardContextProvider, self.VCARD_CONTEXT_PROVIDERS) ##[ Pluggable cron jobs ]################################################# FAST_PERIODIC_JOBS = {} SLOW_PERIODIC_JOBS = {} def register_fast_periodic_job(self, name, period, callback): self._compat_check() # FIXME: complain about duplicates? self.FAST_PERIODIC_JOBS[name] = (period, callback) def register_slow_periodic_job(self, name, period, callback): self._compat_check() # FIXME: complain about duplicates? self.SLOW_PERIODIC_JOBS[name] = (period, callback) ##[ Pluggable background worker threads ]################################ WORKERS = [] def register_worker(self, thread_obj): self._compat_check() safe_assert(hasattr(thread_obj, 'start')) safe_assert(hasattr(thread_obj, 'quit')) # FIXME: complain about duplicates? self.WORKERS.append(thread_obj) ##[ Pluggable commands ]################################################## def register_commands(self, *args): self._compat_check() COMMANDS = mailpile.commands.COMMANDS for cls in args: if cls not in COMMANDS: COMMANDS.append(cls) ##[ Pluggable javascript, CSS template and static content ]############### JS_CLASSES = {} CSS_FILES = {} WEB_ASSETS = {} def register_js(self, plugin, classname, filename): self.JS_CLASSES['%s.%s' % (plugin, classname)] = filename def register_css(self, plugin, classname, filename): self.CSS_FILES['%s.%s' % (plugin, classname)] = filename def register_web_asset(self, plugin, path, filename, mimetype='text/html'): if path in self.WEB_ASSETS: raise PluginError(_('Already registered: %s') % path) self.WEB_ASSETS[path] = (filename, mimetype, plugin) def get_js_classes(self): return self.JS_CLASSES def get_css_files(self): return self.CSS_FILES def get_web_asset(self, path, default=None): return tuple(self.WEB_ASSETS.get(path, [default, None])[0:2]) ##[ Pluggable UI elements ]############################################### # These are the elements that exist at the moment UI_ELEMENTS = { 'settings': [], 'activities': [], 'email_activities': [], # Activities on e-mails 'thread_activities': [], # Activities on e-mails in a thread 'display_modes': [], 'display_refiners': [], 'selection_actions': [] } def register_ui_element(self, ui_type, context=None, name=None, text=None, icon=None, description=None, url=None, javascript_setup=None, javascript_events=None, **kwargs): name = name.replace('/', '_') if name not in [e.get('name') for e in self.UI_ELEMENTS[ui_type]]: # FIXME: Is context valid? info = { "context": context or [], "name": name, "text": text, "icon": icon, "description": description, "javascript_setup": javascript_setup, "javascript_events": javascript_events, "url": url } for k, v in kwargs.iteritems(): info[k] = v self.UI_ELEMENTS[ui_type].append(info) else: raise ValueError('Duplicate element: %s' % name) def get_ui_elements(self, ui_type, context): # FIXME: This is a bit inefficient. # The good thing is, it maintains a stable order. return [elem for elem in self.UI_ELEMENTS[ui_type] if context in elem['context']] ##[ Backwards compatibility ]################################################ _default_pm = PluginManager(builtin=False, deprecated=True) register_config_variables = _default_pm.register_config_variables register_config_section = _default_pm.register_config_section register_data_kw_extractor = _default_pm.register_data_kw_extractor register_text_kw_extractor = _default_pm.register_text_kw_extractor register_meta_kw_extractor = _default_pm.register_meta_kw_extractor get_data_kw_extractors = _default_pm.get_data_kw_extractors get_text_kw_extractors = _default_pm.get_text_kw_extractors get_meta_kw_extractors = _default_pm.get_meta_kw_extractors get_search_term = _default_pm.get_search_term register_search_term = _default_pm.register_search_term filter_hooks = _default_pm.get_filter_hooks register_filter_hook_pre = _default_pm.register_filter_hook_pre register_filter_hook_post = _default_pm.register_filter_hook_post register_vcard_importers = _default_pm.register_vcard_importers register_contact_exporters = _default_pm.register_contact_exporters register_contact_context_providers = _default_pm.register_contact_context_providers register_fast_periodic_job = _default_pm.register_fast_periodic_job register_slow_periodic_job = _default_pm.register_slow_periodic_job register_worker = _default_pm.register_worker register_commands = _default_pm.register_commands ================================================ FILE: mailpile/plugins/autotag.py ================================================ # This is the generic auto-tagging plugin. # # We feed the classifier the same terms as go into the search engine, # which should allow us to actually introspect a bit into the behavior # of the classifier. import math import time import datetime import mailpile.util from mailpile.commands import Command from mailpile.config.base import ConfigDict from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailutils.emails import Email from mailpile.plugins import PluginManager from mailpile.util import * _plugins = PluginManager(builtin=__file__) ##[ Configuration ]########################################################### TAGGERS = {} TRAINERS = {} AUTO_TAG_DISABLED = (None, False, '', 'off', 'false', 'fancy', 'builtin') AUTO_TAG_CONFIG = { 'match_tag': ['Tag we are adding to automatically', str, ''], 'unsure_tag': ['If unsure, add to this tag', str, ''], 'exclude_tags': ['Tags on messages we should never match (ham)', str, []], 'ignore_kws': ['Ignore messages with these keywords', str, []], 'corpus_size': ['How many messages do we train on?', int, 1200], 'threshold': ['Size of the sure/unsure ranges', float, 0.1], 'tagger': ['Internal class name or |shell command', str, ''], 'trainer': ['Internal class name or |shell commant', str, '']} _plugins.register_config_section( 'prefs', 'autotag', ["Auto-tagging", AUTO_TAG_CONFIG , []]) def at_identify(at_config): return md5_hex(at_config.match_tag, at_config.tagger, at_config.trainer)[:12] def autotag_configs(config): done = [] for at_config in config.prefs.autotag: yield at_config done.append(at_config.match_tag) taggers = [k for k in TAGGERS.keys() if k != '_default'] if not taggers: return for tid, tag_info in config.tags.iteritems(): auto_tagging = (tag_info.auto_tag or '') if (tid not in done and auto_tagging.lower() not in AUTO_TAG_DISABLED): at_config = ConfigDict(_rules=AUTO_TAG_CONFIG) at_config.match_tag = tid if auto_tagging not in taggers: auto_tagging = taggers[0] at_config.tagger = auto_tagging at_config.trainer = auto_tagging yield at_config class AutoTagger(object): def __init__(self, tagger, trainer): self.tagger = tagger self.trainer = trainer self.trained = False def reset(self, at_config): """Reset to an untrained state""" self.trainer.reset(self, at_config) self.trained = False def learn(self, *args): self.trained = True return self.trainer.learn(self, *args) def should_tag(self, *args): return self.tagger.should_tag(self, *args) def SaveAutoTagger(config, at_config): aid = at_identify(at_config) at = config.autotag.get(aid) if at and at.trained: config.save_pickle(at, 'pickled-autotag.%s' % aid) def LoadAutoTagger(config, at_config): if not config.real_hasattr('autotag'): config.real_setattr('autotag', {}) aid = at_identify(at_config) at = config.autotag.get(aid) if aid not in config.autotag: cfn = 'pickled-autotag.%s' % aid try: config.autotag[aid] = config.load_pickle(cfn) except (IOError, EOFError): tagger = at_config.tagger trainer = at_config.trainer config.autotag[aid] = AutoTagger( TAGGERS.get(tagger, TAGGERS['_default'])(tagger), TRAINERS.get(trainer, TRAINERS['_default'])(trainer)) SaveAutoTagger(config, at_config) return config.autotag[aid] # FIXME: This is dumb import mailpile.config.manager mailpile.config.manager.ConfigManager.load_auto_tagger = LoadAutoTagger mailpile.config.manager.ConfigManager.save_auto_tagger = SaveAutoTagger ##[ Internal classes ]######################################################## class AutoTagCommand(object): def __init__(self, command): self.command = command class Tagger(AutoTagCommand): def should_tag(self, atagger, at_config, msg, keywords): """Returns (result, evidence), result =True, False or None""" return (False, None) class Trainer(AutoTagCommand): def learn(self, atagger, at_config, msg, keywords, should_tag): """Learn that this message should (or should not) be tagged""" pass def reset(self, atagger, at_config): """Reset to an untrained state (called by AutoTagger.reset)""" pass TAGGERS['_default'] = Tagger TRAINERS['_default'] = Trainer ##[ Commands ]################################################################ class AutoTagCommand(Command): ORDER = ('Tagging', 9) def _get_keywords(self, e): idx = self._idx() if not hasattr(self, 'rcache'): self.rcache = {} mid = e.msg_mid() if mid not in self.rcache: kws, snippet = idx.read_message( self.session, mid, e.get_msg_info(field=idx.MSG_ID), e.get_msg(), e.get_msg_size(), int(e.get_msg_info(field=idx.MSG_DATE), 36)) self.rcache[mid] = kws return self.rcache[mid] class Retrain(AutoTagCommand): SYNOPSIS = (None, 'autotag/retrain', None, '[]') def command(self): return self._retrain(tags=self.args) def _retrain(self, tags=None): "Retrain autotaggers" session, config, idx = self.session, self.session.config, self._idx() tags = tags or [asb.match_tag for asb in autotag_configs(config)] tids = [config.get_tag(t)._key for t in tags if t] session.ui.mark(_('Retraining SpamBayes autotaggers')) if not config.real_hasattr('autotag'): config.real_setattr('autotag', {}) # Find all the interesting messages! We don't look in the trash, # but we do look at interesting spam. # # Note: By specifically stating that we DON'T want trash, we # disable the search engine's default result suppression # and guarantee these results don't corrupt the somewhat # lame/broken result cache. # no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')] interest = {} for ttype in ('replied', 'read', 'tagged'): interest[ttype] = set() for tag in config.get_tags(type=ttype): interest[ttype] |= idx.search(session, ['in:%s' % tag.slug] + no_trash ).as_set() session.ui.notify(_('Have %d interesting %s messages' ) % (len(interest[ttype]), ttype)) retrained, unreadable = [], [] count_all = 0 for at_config in autotag_configs(config): at_tag = config.get_tag(at_config.match_tag) if at_tag and at_tag._key in tids: session.ui.mark('Retraining: %s' % at_tag.name) yn = [(set(), set(), 'in:%s' % at_tag.slug, True), (set(), set(), '-in:%s' % at_tag.slug, False)] # Get the current message sets: tagged and untagged messages # excluding trash. for tset, mset, srch, which in yn: mset |= idx.search(session, [srch] + no_trash).as_set() # If we have any exclude_tags, they are particularly # interesting, so we'll look at them first. interesting = [] for etagid in at_config.exclude_tags: etag = config.get_tag(etagid) if etag._key not in interest: srch = ['in:%s' % etag._key] + no_trash interest[etag._key] = idx.search(session, srch ).as_set() interesting.append(etag._key) interesting.extend(['replied', 'read', 'tagged', None]) # Go through the interest types in order of preference and # while we still lack training data, add to the training set. for ttype in interesting: for tset, mset, srch, which in yn: # False positives are really annoying, and generally # speaking any autotagged subset should be a small # part of the Universe. So we divide the corpus # budget 33% True, 67% False. full_size = int(at_config.corpus_size * (0.33 if which else 0.67)) want = min(full_size // len(interesting), max(0, full_size - len(tset))) # Make sure we always fully utilize our budget if full_size > len(tset) and not ttype: want = full_size - len(tset) if want: if ttype: adding = sorted(list(mset & interest[ttype])) else: adding = sorted(list(mset)) adding = set(list(reversed(adding))[:want]) tset |= adding mset -= adding # Load classifier, reset atagger = config.load_auto_tagger(at_config) atagger.reset(at_config) for tset, mset, srch, which in yn: count = 0 # We go through the list of message in order, to avoid # thrashing caches too badly. for msg_idx in sorted(list(tset)): try: e = Email(idx, msg_idx) count += 1 count_all += 1 session.ui.mark( _('Reading %s (%d/%d, %s=%s)' ) % (e.msg_mid(), count, len(tset), at_tag.name, which)) atagger.learn(at_config, e.get_msg(), self._get_keywords(e), which) play_nice_with_threads() if mailpile.util.QUITTING: return self._error('Aborted') except (IndexError, TypeError, ValueError, OSError, IOError): if 'autotag' in session.config.sys.debug: import traceback traceback.print_exc() unreadable.append(msg_idx) session.ui.warning( _('Failed to process message at =%s' ) % (b36(msg_idx))) # We got this far without crashing, so save the result. config.save_auto_tagger(at_config) retrained.append(at_tag.name) message = _('Retrained SpamBayes auto-tagging for %s' ) % ', '.join(retrained) session.ui.mark(message) return self._success(message, result={ 'retrained': retrained, 'unreadable': unreadable, 'read_messages': count_all }) @classmethod def interval_retrain(cls, session): """ Retrains autotaggers Classmethod used for periodic automatic retraining """ result = cls(session)._retrain() if result: return True else: return False _plugins.register_config_variables('prefs', { 'autotag_retrain_interval': [ _('Periodically retrain autotagger (seconds)'), int, 24*60*60]}) _plugins.register_slow_periodic_job( 'retrain_autotag', 'prefs.autotag_retrain_interval', Retrain.interval_retrain) class Classify(AutoTagCommand): SYNOPSIS = (None, 'autotag/classify', None, '') ORDER = ('Tagging', 9) def _classify(self, emails): session, config, idx = self.session, self.session.config, self._idx() results = {} unknown = [] for e in emails: kws = self._get_keywords(e) result = results[e.msg_mid()] = {} for at_config in autotag_configs(config): if not at_config.match_tag: continue at_tag = config.get_tag(at_config.match_tag) if not at_tag and at_config.match_tag not in unknown: session.ui.error(_('Unknown tag: %s' ) % at_config.match_tag) unknown.append(at_config.match_tag) continue atagger = config.load_auto_tagger(at_config) if atagger.trained: result[at_tag._key] = result.get(at_tag._key, []) result[at_tag._key].append(atagger.should_tag( at_config, e.get_msg(), kws )) return results def command(self): session, config, idx = self.session, self.session.config, self._idx() emails = [Email(idx, mid) for mid in self._choose_messages(self.args)] return self._success(_('Classified %d messages') % len(emails), self._classify(emails)) class AutoTag(Classify): SYNOPSIS = (None, 'autotag', None, '') ORDER = ('Tagging', 9) def command(self): session, config, idx = self.session, self.session.config, self._idx() emails = [Email(idx, mid) for mid in self._choose_messages(self.args)] scores = self._classify(emails) tag = {} for mid in scores: for at_config in autotag_configs(config): at_tag = config.get_tag(at_config.match_tag) if not at_tag: continue wants = scores[mid].get(at_tag._key, [(False, )]) want = bool([True for w in wants if w[0]]) if want is True: if at_config.match_tag not in tag: tag[at_config.match_tag] = [mid] else: tag[at_config.match_tag].append(mid) elif at_config.unsure_tag and want is None: if at_config.unsure_tag not in tag: tag[at_config.unsure_tag] = [mid] else: tag[at_config.unsure_tag].append(mid) for tid in tag: idx.add_tag(session, tid, msg_idxs=[int(i, 36) for i in tag[tid]]) return self._success(_('Auto-tagged %d messages') % len(emails), tag) _plugins.register_commands(Retrain, Classify, AutoTag) ##[ Keywords ]################################################################ def filter_hook(session, msg_mid, msg, keywords, **kwargs): """Classify this message.""" if not kwargs.get('incoming', False): return keywords config = session.config for at_config in autotag_configs(config): try: at_tag = config.get_tag(at_config.match_tag) atagger = config.load_auto_tagger(at_config) if not atagger.trained: continue want, info = atagger.should_tag(at_config, msg, keywords) if want is True: if 'autotag' in config.sys.debug: session.ui.debug(('Autotagging %s with %s (w=%s, i=%s)' ) % (msg_mid, at_tag.name, want, info)) keywords.add('%s:in' % at_tag._key) elif at_config.unsure_tag and want is None: unsure_tag = config.get_tag(at_config.unsure_tag) if 'autotag' in config.sys.debug: session.ui.debug(('Autotagging %s with %s (w=%s, i=%s)' ) % (msg_mid, unsure_tag.name, want, info)) keywords.add('%s:in' % unsure_tag._key) except (KeyError, AttributeError, ValueError): pass return keywords # We add a filter pre-hook with a high (late) priority. Late priority to # maximize the amount of data we are feeding to the classifier, but a # pre-hook so normal filter rules will override the autotagging. _plugins.register_filter_hook_pre('90-autotag', filter_hook) ================================================ FILE: mailpile/plugins/autotag_sb.py ================================================ # Add SpamBayes as an option to the autotagger. We like SpamBayes. # # We feed the classifier the same terms as go into the search engine, # which should allow us to actually introspect a bit into the behavior # of the classifier. from mailpile.spambayes import Classifier import mailpile.plugins.autotag from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n def _classifier(autotagger): if not hasattr(autotagger, 'spambayes'): autotagger.spambayes = Classifier() return autotagger.spambayes class SpamBayesTagger(mailpile.plugins.autotag.Trainer): def should_tag(self, atagger, at_config, msg, keywords): score, evidence = _classifier(atagger).chi2_spamprob(keywords, evidence=True) if score >= 1 - at_config.threshold: want = True elif score > at_config.threshold: want = None else: want = False return (want, score) class SpamBayesTrainer(mailpile.plugins.autotag.Trainer): def learn(self, atagger, at_config, msg, keywords, should_tag): _classifier(atagger).learn(keywords, should_tag) def reset(self, atagger, at_config): atagger.spambayes = Classifier() mailpile.plugins.autotag.TAGGERS['spambayes'] = SpamBayesTagger mailpile.plugins.autotag.TRAINERS['spambayes'] = SpamBayesTrainer ================================================ FILE: mailpile/plugins/backups.py ================================================ from __future__ import print_function import cStringIO import datetime import gzip import json import os import sys import time import traceback import urllib import zipfile from mailpile.auth import VerifyAndStorePassphrase from mailpile.config.defaults import APPVER from mailpile.commands import Command from mailpile.crypto.streamer import EncryptingStreamer, DecryptingStreamer from mailpile.plugins import PluginManager from mailpile.plugins.core import Quit from mailpile.i18n import ActivateTranslation from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.ui import SuppressHtmlOutput from mailpile.util import * from mailpile.vfs import FilePath, vfs _ = lambda t: t _plugins = PluginManager(builtin=__file__) def _gzip(filename, data): gzip_data = cStringIO.StringIO() gzip_obj = gzip.GzipFile(filename, 'w', 9, gzip_data, 0) gzip_obj.write(data) gzip_obj.close() return gzip_data.getvalue() def _gunzip(data): with gzip.GzipFile('', 'rb', 0, cStringIO.StringIO(data)) as gzf: return gzf.read() def _decrypt(data, config): with DecryptingStreamer(cStringIO.StringIO(data), mep_key=config.get_master_key()) as fd: data = fd.read() fd.verify(_raise=IOError) return data class MakeBackup(Command): """Generate an encrypted backup of Stuff""" SYNOPSIS = (None, 'backup', 'backup', '[download]') ORDER = ('Internals', 6) RAISES = (SuppressHtmlOutput,) CONFIG_REQUIRED = True IS_USER_ACTIVITY = False @classmethod def SummarizeTags(cls, config): # First, decide which tags to include. # Not all tags are interesting! Most, but not all. keep = {} suppress = {} for tid, tag in config.tags.iteritems(): if tag.type in ('tag', 'group', 'attribute', 'inbox', 'drafts', 'sent', 'spam', 'read', 'tagged', 'fwded', 'replied', 'search', 'profile'): if tid in config.index.TAGS: keep[tid] = tag elif tag.type == 'trash': suppress[tid] = tag msg_idx_set = set() for tid in keep: msg_idx_set |= config.index.TAGS[tid] for tid in suppress: msg_idx_set -= config.index.TAGS.get(tid, set([])) msg_id_list = [''] * len(config.index.INDEX) for msgid, msg_idx in config.index.MSGIDS.iteritems(): if msg_idx in msg_idx_set: msg_id_list[msg_idx] = msgid return { 'tags': dict((tid, list(config.index.TAGS[tid])) for tid in keep), 'msgids': msg_id_list} @classmethod def MakeBackupArchive(cls, config, gnupg, what=None): backup_date = datetime.date.today().strftime('%Y-%m-%d') if what: backup_fn = 'Mailpile_Backup_%s_%s.zip' % ( backup_date, ','.join(what)) else: backup_fn = 'Mailpile_Backup_%s.zip' % (backup_date,) # Prep archive! backup_data = cStringIO.StringIO() backup_zip = zipfile.ZipFile(backup_data, 'w', zipfile.ZIP_DEFLATED) backup_zip.writestr('README.txt', (('\n'.join([ _("This is a backup of Mailpile v%(ver)s keys and configuration."), '', ' * ' + _("This backup was generated on: %(date)s."), ' * ' + _("The contents of this file should be encrypted."), ' * ' + _("The entire ZIP file must be uploaded during " "restoration."), '', '-- ', '{"backup_date": "%(date)s",', ' "backup_version": 1.0,', ' "mailpile_version": "%(ver)s"}' ])) % {'ver': APPVER, 'date': backup_date}).strip()) backup_contents = [] def _add_file(realfile, zipname): backup_zip.write(realfile, zipname) backup_contents.append(zipname) # The .ZIP is unencrypted, so generated contents needs protecting def _encrypt_and_add_data(filename, data): tempfile = os.path.join(config.tempfile_dir(), filename) with EncryptingStreamer(config.get_master_key(), dir=config.tempfile_dir()) as fd: fd.write(data) fd.save(tempfile) _add_file(tempfile, filename) safe_remove(tempfile) # What has been requested? if what and what[0] == 'full': what += ['config', 'profiles', 'keys', 'gnupg', 'vcards', 'tags'] # Critical: Copy the configuration and master keys if not what or 'config' in what: for fn in (config.conf_pub, config.conf_key, config.conffile): _add_file(fn, os.path.basename(fn)) # Critical: Copy the profile VCard data if not what or 'profiles' in what: for profile in config.vcards.find_vcards([], kinds=['profile']): target = os.path.basename(profile.filename) _add_file(profile.filename, os.path.join('vcards', target)) # Critical: Copy all the private GnuPG keys! if not what or 'keys' in what: _encrypt_and_add_data('gnupg-privkeys.asc.gze', _gzip('gnupg-privkeys.asc', gnupg.export_privkeys())) # Recommended: Copy all the public GnuPG keys! if not what or 'gnupg' in what: _encrypt_and_add_data('gnupg-pubkeys.asc.gze', _gzip('gnupg-pubkeys.asc', gnupg.export_pubkeys())) # Recommended: Copy the "interesting" VCards. if not what or 'vcards' in what: for vcard in config.vcards.find_vcards([], kinds=['individual', 'group']): if ((what and 'full' in what) or vcard.recent_history() or vcard.crypto_policy or vcard.html_policy or vcard.pgp_key_shared or vcard.pgp_key): target = os.path.basename(vcard.filename) _add_file(vcard.filename, os.path.join('vcards', target)) # Optional: Backup the tag structure. This is useful if we lose the # metadata index, but have the original e-mails. This is DISABLED BY # DEFAULT because it is expensive and that may not be a real use case. if what and 'tags' in what: _encrypt_and_add_data('tags.json.gze', _gzip('tags.json', json.dumps(cls.SummarizeTags(config)))) # Finalize archive backup_zip.close() backup_data = backup_data.getvalue() return backup_fn, backup_contents, backup_data def command(self): session, config = self.session, self.session.config html_variables = session.ui.html_variables if not (html_variables and session.ui.valid_csrf_token(self.data.get('csrf', [''])[0])): raise AccessError('Invalid CSRF token') backup_fn, backup_contents, backup_data = self.MakeBackupArchive( config, self._gnupg(), what=[a for a in self.args if a not in ('download',)]) if 'download' in self.args: encoded_fn = urllib.quote(backup_fn.encode('utf-8')) request = html_variables['http_request'] request.send_http_response(200, 'OK') request.send_standard_headers(mimetype='application/zip', header_list=[ ('Content-Length', len(backup_data)), ('Content-Disposition', 'attachment; filename*=UTF-8\'\'%s' % (encoded_fn,))]) request.wfile.write(backup_data) raise SuppressHtmlOutput() return self._success('Generated backup', result={ 'filename': backup_fn, 'contents': backup_contents, 'data_b64': backup_data.encode('base64')}) AVAILABLE_BACKUPS = {} class RestoreBackup(Command): """Bootstraup setup from a backup archive""" SYNOPSIS = (None, 'backup/restore', 'backup/restore', '[/path/to.zip]') ORDER = ('Internals', 6) RAISES = (UrlRedirectException,) CONFIG_REQUIRED = False HTTP_AUTH_REQUIRED = 'maybe' HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = { 'lang': 'Language to use in UI'} HTTP_POST_VARS = { 'restore': 'date of backup to restore', 'password': 'Mailpile master password', 'keychain': 'GnuPG keychain policy: shared*, mailpile, none', 'os_settings': 'OS settings policy: keep, backup*', 'file-data': 'file data'} def _restore_PGP_keys(self, config, backup_zip, policy): if policy not in ('shared', 'mailpile'): return if policy == 'mailpile': config.sys.gpg_home = config.workdir else: config.sys.gpg_home = '' for keyfile in ('gnupg-pubkeys.asc.gze', 'gnupg-privkeys.asc.gze'): gze = backup_zip.read(keyfile) print('DATA: %s' % gze) self._gnupg().import_keys(_gunzip(_decrypt(gze, config))) def _adjust_paths(self, config): # Go through sys.mailboxes, sources.*.mailbox: # - if the path is outside Workdir, does not exist, clear entry # - if the path is inside Workdir, does not exist, create it # - if the path is src:, source does not exist, clear entry def path_ok(mbx_path): if 'src:' in mbx_path.raw_fp[:5]: return True elif vfs.mailbox_type(mbx_path, config): return True elif unicode(mbx_path).startswith('/Mailpile$/'): config.create_local_mailstore( self.session, name=mbx_path.raw_fp) return True else: return False for i, mbx_path in config.sys.mailbox.iteritems(): mbx_path = FilePath(mbx_path) if not path_ok(mbx_path): config.sys.mailbox[i] = '/dev/null' for i, p, src in config.get_mailboxes(with_mail_source=True, mail_source_locals=True): mbx_path = FilePath(p) if src.mailbox[i].local and not path_ok(mbx_path): src.mailbox[i].local = '!CREATE' def command(self): global AVAILABLE_BACKUPS session, config = self.session, self.session.config message, results = '', {} if config.prefs.gpg_recipient or os.path.exists(config.conf_key): raise UrlRedirectException('/' + (config.sys.http_path or '')) if 'lang' in self.data: ActivateTranslation(session, config, self.data['lang'][0]) password = '' if self.args and '_method' not in self.data: try: if self.args[0] in AVAILABLE_BACKUPS: backup_data = AVAILABLE_BACKUPS[self.args[0]] self.data['restore'] = [self.args[0]] password = session.ui.get_password(_("Your password: ")) else: with open(self.args[0], 'r') as fd: backup_data = fd.read() except (IOError, OSError): return self._error('Failed to read: %s' % self.args[0]) elif self.data.get('_method') == 'POST': if 'restore' in self.data: backup_data = AVAILABLE_BACKUPS[self.data['restore'][0]] password = self.data.get('password', [''])[0] else: backup_data = self.data.get('file-data', [None])[0] else: backup_data = None if backup_data is not None: try: if isinstance(backup_data, str): backup_data = cStringIO.StringIO(backup_data) backup_zip = zipfile.ZipFile(backup_data, 'r') # Load and validate metadata (from README.txt) results['metadata'] = metadata = json.loads( backup_zip.read('README.txt').split('-- ')[1]) results['metadata']['contents'] = backup_zip.namelist() backup_date = metadata['backup_date'] if metadata['backup_version'] != 1.0: raise ValueError('Unrecognized backup version') # If we get this far, the backup looks good. Restore? if (password and backup_date == self.data.get('restore', [''])[0]): # This should be safe: we are in the setup phase where # almost no background stuff is running, so it should be # fine to just overwrite files and reload. config.stop_workers() backup_zip.extractall(config.workdir) VerifyAndStorePassphrase(config, password) os_gpg_home = config.sys.gpg_home os_gpg_binary = config.sys.gpg_binary os_http_port = config.sys.http_port os_minfree_mb = config.sys.minfree_mb try: config.load(session) except IOError: pass B = ['backup'] if 'keep' == self.data.get('os_settings', B)[0]: config.sys.gpg_home = os_gpg_home config.sys.gpg_binary = os_gpg_binary config.sys.http_port = os_http_port config.sys.minfree_mb = os_minfree_mb self._restore_PGP_keys(config, backup_zip, self.data.get('keychain', ['shared'])[0]) self._adjust_paths(config) config.prepare_workers(session, daemons=True) message = _('Backup restored') results['restored'] = True AVAILABLE_BACKUPS = {} else: message = _('Backup validated, restoration is possible') AVAILABLE_BACKUPS[backup_date] = backup_data except (ValueError, KeyError, zipfile.BadZipfile, IOError): traceback.print_exc() return self._error('Incomplete, invalid or corrupt backup') else: message = _('Restore from backup') results['available'] = AVAILABLE_BACKUPS.keys() return self._success(message, result=results) _plugins.register_commands(MakeBackup, RestoreBackup) ================================================ FILE: mailpile/plugins/compose.py ================================================ import datetime import email.utils import os import os.path import re import traceback import mailpile.security as security from mailpile.commands import Command from mailpile.crypto.state import * from mailpile.crypto.mime import EncryptionFailureError, SignatureFailureError from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.plugins import PluginManager from mailpile.mailutils import NoFromAddressError, NotEditableError from mailpile.mailutils.addresses import AddressHeaderParser from mailpile.mailutils.emails import ExtractEmailAndName, Email from mailpile.mailutils.emails import PrepareMessage, MakeMessageID from mailpile.search import MailIndex from mailpile.smtp_client import SendMail from mailpile.urlmap import UrlMap from mailpile.util import * from mailpile.vcard import AddressInfo from mailpile.plugins.search import Search, SearchResults, View GLOBAL_EDITING_LOCK = MboxRLock() _plugins = PluginManager(builtin=__file__) class EditableSearchResults(SearchResults): def __init__(self, session, idx, new, sent, **kwargs): SearchResults.__init__(self, session, idx, **kwargs) self.new_messages = new self.sent_messages = sent if new: self['created'] = [m.msg_mid() for m in new] if sent: self['sent'] = [m.msg_mid() for m in new] self['summary'] = _('Sent: %s') % self['summary'] def AddComposeMethods(cls): class newcls(cls): COMMAND_CACHE_TTL = 0 COMMAND_SECURITY = security.CC_COMPOSE_EMAIL def _create_contacts(self, emails): try: from mailpile.plugins.contacts import AddContact AddContact(self.session, arg=['=%s' % e.msg_mid() for e in emails] ).run(recipients=True, quietly=True, internal=True) except (TypeError, ValueError, IndexError): self._ignore_exception() def _tag_emails(self, emails, tag): try: idx = self._idx() idx.add_tag(self.session, self.session.config.get_tag_id(tag), msg_idxs=[e.msg_idx_pos for e in emails], conversation=False) except (TypeError, ValueError, IndexError): self._ignore_exception() def _untag_emails(self, emails, tag): try: idx = self._idx() idx.remove_tag(self.session, self.session.config.get_tag_id(tag), msg_idxs=[e.msg_idx_pos for e in emails], conversation=False) except (TypeError, ValueError, IndexError): self._ignore_exception() def _tagger(self, emails, untag, **kwargs): tag = self.session.config.get_tags(**kwargs) if tag and untag: return self._untag_emails(emails, tag[0]._key) elif tag: return self._tag_emails(emails, tag[0]._key) def _tag_blank(self, emails, untag=False): return self._tagger(emails, untag, type='blank') def _tag_drafts(self, emails, untag=False): return self._tagger(emails, untag, type='drafts') def _tag_outbox(self, emails, untag=False): return self._tagger(emails, untag, type='outbox') def _tag_sent(self, emails, untag=False): return self._tagger(emails, untag, type='sent') def _track_action(self, action_type, refs): session, idx = self.session, self._idx() for tag in session.config.get_tags(type=action_type): idx.add_tag(session, tag._key, msg_idxs=[m.msg_idx_pos for m in refs]) def _actualize_ephemeral(self, ephemeral_mid): idx = self._idx() if isinstance(ephemeral_mid, int): # Not actually ephemeral, just return a normal Email return Email(idx, ephemeral_mid) msgid, mid = ephemeral_mid.rsplit('-', 1) etype, etarg, msgid = msgid.split('-', 2) if etarg not in ('all', 'att'): msgid = etarg + '-' + msgid msgid = '<%s>' % msgid.replace('_', '@') etype = etype.lower() enc_msgid = idx._encode_msg_id(msgid) msg_idx = idx.MSGIDS.get(enc_msgid) if msg_idx is not None: # Already actualized, just return a normal Email return Email(idx, msg_idx) if etype == 'forward': refs = [Email(idx, int(mid, 36))] e = Forward.CreateForward(idx, self.session, refs, msgid, with_atts=(etarg == 'att'))[0] self._track_action('fwded', refs) elif etype == 'reply': refs = [Email(idx, int(mid, 36))] e = Reply.CreateReply(idx, self.session, refs, msgid, reply_all=(etarg == 'all'))[0] self._track_action('replied', refs) else: e = Compose.CreateMessage(idx, self.session, msgid)[0] self._tag_blank([e]) self.session.ui.debug('Actualized: %s' % e.msg_mid()) return Email(idx, e.msg_idx_pos) return newcls class CompositionCommand(AddComposeMethods(Search)): HTTP_QUERY_VARS = {} HTTP_POST_VARS = {} UPDATE_STRING_DATA = { 'mid': 'metadata-ID', 'subject': '..', 'from': '..', 'to': '..', 'cc': '..', 'bcc': '..', 'body': '..', 'encryption': '..', 'attachment': '..', 'attach-pgp-pubkey': '..', } UPDATE_HEADERS = ('Subject', 'From', 'To', 'Cc', 'Bcc', 'Encryption', 'Attach-PGP-Pubkey') def _new_msgid(self): msgid = (MakeMessageID() .replace('.', '-') # Dots may bother JS/CSS .replace('_', '-')) # We use _ to encode the @ later on return msgid def _get_email_updates(self, idx, create=False, noneok=False, emails=None): # Split the argument list into files and message IDs files = [f[1:].strip() for f in self.args if f.startswith('<')] args = [a for a in self.args if not a.startswith('<')] # Message IDs can come from post data for mid in self.data.get('mid', []): args.append('=%s' % mid) emails = emails or [self._actualize_ephemeral(mid) for mid in self._choose_messages(args, allow_ephemeral=True)] update_header_set = (set(self.data.keys()) & set([k.lower() for k in self.UPDATE_HEADERS])) updates, fofs = [], 0 for e in (emails or (create and [None]) or []): # If we don't have a file, check for posted data if len(files) not in (0, 1, len(emails)): return (self._error(_('Cannot update from multiple files')), None) elif len(files) == 1: updates.append((e, self._read_file_or_data(files[0]))) elif files and (len(files) == len(emails)): updates.append((e, self._read_file_or_data(files[fofs]))) elif update_header_set: # No file name, construct an update string from the POST data. etree = e and e.get_message_tree() or {} defaults = etree.get('editing_strings', {}) up = [] for hdr in self.UPDATE_HEADERS: if hdr.lower() in self.data: data = ', '.join(self.data[hdr.lower()]) else: data = defaults.get(hdr.lower(), '') up.append('%s: %s' % (hdr, data)) # This preserves in-reply-to, references and any other # headers we're not usually keen on editing. if defaults.get('headers'): up.append(defaults['headers']) # This weird thing converts attachment=1234:bla.txt into a # dict of 1234=>bla.txt values, attachment=1234 to 1234=>None. # .. or just keeps all attachments if nothing is specified. att_keep = (dict([(ai.split(':', 1) if (':' in ai) else (ai, None)) for ai in self.data.get('attachment', [])]) if 'attachment' in self.data else defaults.get('attachments', {})) for att_id, att_fn in defaults.get('attachments', {}).iteritems(): if att_id in att_keep: fn = att_keep[att_id] or att_fn up.append('Attachment-%s: %s' % (att_id, fn)) updates.append((e, '\n'.join( up + ['', '\n'.join(self.data.get('body', defaults.get('body', '')))] ))) elif noneok: updates.append((e, None)) elif 'compose' in self.session.config.sys.debug: sys.stderr.write('Doing nothing with %s' % update_header_set) fofs += 1 if 'compose' in self.session.config.sys.debug: for e, up in updates: sys.stderr.write(('compose/update: Update %s with:\n%s\n--\n' ) % ((e and e.msg_mid() or '(new'), up)) if not updates: sys.stderr.write('compose/update: No updates!\n') return updates def _return_search_results(self, message, emails, expand=None, new=[], sent=[], ephemeral=False, error=None): session, idx = self.session, self._idx() if not ephemeral: session.results = [e.msg_idx_pos for e in emails] else: session.results = ephemeral session.displayed = EditableSearchResults(session, idx, new, sent, results=session.results, num=len(emails), emails=expand) if error: return self._error(message, result=session.displayed, info=error) else: return self._success(message, result=session.displayed) def _edit_messages(self, *args, **kwargs): try: return self._real_edit_messages(*args, **kwargs) except NotEditableError: return self._error(_('Message is not editable')) def _real_edit_messages(self, emails, new=True, tag=True, ephemeral=False): session, idx = self.session, self._idx() if (not ephemeral and (session.ui.edit_messages(session, emails) or not new)): if tag: self._tag_blank(emails, untag=True) self._tag_drafts(emails) self.message = _('%d message(s) edited') % len(emails) else: self.message = _('%d message(s) created') % len(emails) self._background_save(index=True) session.ui.mark(self.message) return self._return_search_results(self.message, emails, expand=emails, new=(new and emails), ephemeral=ephemeral) class Draft(AddComposeMethods(View)): """Edit an existing draft""" SYNOPSIS = ('E', 'edit', 'message/draft', '[]') ORDER = ('Composing', 0) HTTP_QUERY_VARS = { 'mid': 'metadata-ID' } def _side_effects(self, emails): session, idx = self.session, self._idx() with GLOBAL_EDITING_LOCK: if not emails: session.ui.mark(_('No messages!')) elif session.ui.edit_messages(session, emails): self._tag_blank(emails, untag=True) self._tag_drafts(emails) self._background_save(index=True) self.message = _('%d message(s) edited') % len(emails) else: self.message = _('%d message(s) unchanged') % len(emails) session.ui.mark(self.message) return None class Compose(CompositionCommand): """Create a new blank e-mail for editing""" SYNOPSIS = ('C', 'compose', 'message/compose', "[ephemeral]") ORDER = ('Composing', 0) HTTP_CALLABLE = ('POST', 'GET') HTTP_QUERY_VARS = dict_merge(CompositionCommand.UPDATE_STRING_DATA, { 'cid': 'canned response metadata-ID', }) @classmethod def _get_canned(cls, idx, cid): try: return Email(idx, int(cid, 36) ).get_editing_strings().get('body', '') except (ValueError, IndexError, TypeError, OSError, IOError): traceback.print_exc() # FIXME, ugly return '' @classmethod def CreateMessage(cls, idx, session, msgid, cid=None, ephemeral=False): if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None ephemeral = ['new-E-%s-mail' % msgid[1:-1].replace('@', '_')] profiles = session.config.vcards.find_vcards([], kinds=['profile']) return (Email.Create(idx, local_id, lmbox, save=(not ephemeral), msg_text=(cid and cls._get_canned(idx, cid) or ''), msg_id=msgid, ephemeral_mid=ephemeral and ephemeral[0], use_default_from=(len(profiles) == 1)), ephemeral) def command(self): if 'mid' in self.data: return self._error('Please use update for editing messages') session, idx = self.session, self._idx() cid = self.data.get('cid', [None])[0] ephemeral = (self.args and "ephemeral" in self.args) if self.data.get('_method', 'POST') != 'POST': ephemeral = True email, ephemeral = self.CreateMessage(idx, session, self._new_msgid(), cid=cid, ephemeral=ephemeral) if not ephemeral: self._tag_blank([email]) email_updates = self._get_email_updates(idx, emails=[email], create=True) update_string = email_updates and email_updates[0][1] if update_string: email.update_from_string(session, update_string) return self._edit_messages([email], ephemeral=ephemeral, new=(ephemeral or not update_string)) class RelativeCompose(Compose): _ATT_MIMETYPES = ('application/pgp-signature', ) _TEXT_PARTTYPES = ('text', 'quote', 'pgpsignedtext', 'pgpsecuretext', 'pgpverifiedtext') _FW_REGEXP = re.compile(r'^(fwd|fw):.*', re.IGNORECASE) _RE_REGEXP = re.compile(r'^(rep|re):.*', re.IGNORECASE) @staticmethod def prefix_subject(subject, prefix, prefix_regex): """Avoids stacking several consecutive Fw: Re: Re: Re:""" if subject is None: return prefix elif prefix_regex.match(subject): return subject else: return '%s %s' % (prefix, subject) class Reply(RelativeCompose): """Create reply(-all) drafts to one or more messages""" SYNOPSIS = ('r', 'reply', 'message/reply', '[all|ephemeral] ') ORDER = ('Composing', 3) HTTP_QUERY_VARS = { 'mid': 'metadata-ID', 'cid': 'canned response metadata-ID', 'reply_all': 'reply to all', 'ephemeral': 'ephemerality', } HTTP_POST_VARS = {} @classmethod def _add_gpg_key(cls, idx, session, addr): fe, fn = ExtractEmailAndName(addr) vcard = session.config.vcards.get_vcard(fe) if vcard: keys = vcard.get_all('KEY') if keys: mime, fp = keys[0].value.split('data:')[1].split(',', 1) return "%s <%s#%s>" % (fn, fe, fp) return "%s <%s>" % (fn, fe) @classmethod def _create_from_to_cc(cls, idx, session, trees): config = session.config ahp = AddressHeaderParser() ref_from, ref_to, ref_cc = [], [], [] result = {'from': '', 'to': [], 'cc': []} def merge_contact(ai): vcard = config.vcards.get_vcard(ai.address) if vcard: ai.merge_vcard(vcard) return ai # Parse the headers, so we know what we're working with. We prune # some of the duplicates at this stage. for addrs in [t['addresses'] for t in trees]: alist = [] for dst, addresses in ( (ref_from, addrs.get('reply-to') or addrs.get('from', [])), (ref_to, addrs.get('to', [])), (ref_cc, addrs.get('cc', []))): alist += [d.address for d in dst] dst.extend([a for a in addresses if a.address not in alist]) # 1st, choose a from address. from_ai = config.vcards.choose_from_address( config, ref_from, ref_to, ref_cc) # Note: order matters! if from_ai: result['from'] = ahp.normalized(addresses=[from_ai], force_name=True) def addresses(addrs, exclude=[]): alist = [from_ai.address] if (from_ai) else [] alist += [a.address for a in exclude] return [merge_contact(a) for a in addrs if a.address not in alist and not a.address.startswith('noreply@') and '@noreply' not in a.address] # If only replying to messages sent from chosen from, then this is # a follow-up or clarification, so just use the same headers. if (from_ai and len([e for e in ref_from if e and e.address == from_ai.address]) == len(ref_from)): if ref_to: result['to'] = addresses(ref_to) if ref_cc: result['cc'] = addresses(ref_cc) # Else, if replying to other people: # - Construct To from the From lines, excluding own from # - Construct Cc from the To and CC lines, except new To/From else: result['to'] = addresses(ref_from) result['cc'] = addresses(ref_to + ref_cc, exclude=ref_from) return result @classmethod def CreateReply(cls, idx, session, refs, msgid, reply_all=False, cid=None, ephemeral=False): trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs] headers = cls._create_from_to_cc(idx, session, trees) if not reply_all and 'cc' in headers: del headers['cc'] ref_ids = [t['headers_lc'].get('message-id') for t in trees] ref_subjs = [(t['summary'][4] or t['headers_lc'].get('subject')) for t in trees] msg_bodies = [] for t in trees: # FIXME: Templates/settings for how we quote replies? quoted = ''.join([p['data'] for p in t['text_parts'] if p['type'] in cls._TEXT_PARTTYPES and p['data']]) if quoted: target_width = session.config.prefs.line_length if target_width > 40: quoted = reflow_text(quoted, target_width=target_width-2) text = ((_('%s wrote:') % t['headers_lc']['from']) + '\n' + quoted) msg_bodies.append('\n\n' + text.replace('\n', '\n> ')) if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None fmt = 'reply-all-%s-%s' if reply_all else 'reply-%s-%s' ephemeral = [fmt % (msgid[1:-1].replace('@', '_'), refs[0].msg_mid())] if 'cc' in headers: fmt = _('Composing a reply from %(from)s to %(to)s, cc %(cc)s') else: fmt = _('Composing a reply from %(from)s to %(to)s') session.ui.debug(fmt % headers) extra_headers = [] for tree in trees: try: if 'decrypted' in tree['crypto']['encryption']['status']: extra_headers.append(('x-mp-internal-should-encrypt', 'Y')) extra_headers.append(('Encryption', 'openpgp-sign-encrypt')) break except KeyError: pass if cid: # FIXME: Instead, we should use placeholders in the template # and insert the quoted bits in the right place (or # nowhere if the template doesn't want them). msg_bodies[:0] = [cls._get_canned(idx, cid)] email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=cls.prefix_subject( ref_subjs[-1], 'Re:', cls._RE_REGEXP), msg_from=headers.get('from', None), msg_to=headers.get('to', []), msg_cc=headers.get('cc', []), msg_references=[i for i in ref_ids if i], msg_headers=extra_headers, msg_id=msgid, save=(not ephemeral), ephemeral_mid=ephemeral and ephemeral[0]) return (email, ephemeral) def command(self): session, config, idx = self.session, self.session.config, self._idx() reply_all = False ephemeral = False args = list(self.args) if not args: args = ["=%s" % x for x in self.data.get('mid', [])] ephemeral = truthy((self.data.get('ephemeral') or [False])[0]) reply_all = truthy((self.data.get('reply_all') or [False])[0]) else: while args: if args[0].lower() == 'all': reply_all = args.pop(0) or True elif args[0].lower() == 'ephemeral': ephemeral = args.pop(0) or True else: break # Make sure GET does not change backend state, allow on CLI. if self.data.get('_method', 'POST') != 'POST': ephemeral = True refs = [Email(idx, i) for i in self._choose_messages(args)] if refs: try: cid = self.data.get('cid', [None])[0] email, ephemeral = self.CreateReply(idx, session, refs, self._new_msgid(), reply_all=reply_all, cid=cid, ephemeral=ephemeral) except NoFromAddressError: return self._error(_('You must configure a ' 'From address first.')) if not ephemeral: self._track_action('replied', refs) self._tag_blank([email]) return self._edit_messages([email], ephemeral=ephemeral) else: return self._error(_('No message found')) class Forward(RelativeCompose): """Create forwarding drafts of one or more messages""" SYNOPSIS = ('f', 'forward', 'message/forward', '[att|ephemeral] ') ORDER = ('Composing', 4) HTTP_QUERY_VARS = { 'mid': 'metadata-ID', 'cid': 'canned response metadata-ID', 'ephemeral': 'ephemerality', 'atts': 'forward attachments' } HTTP_POST_VARS = {} @classmethod def CreateForward(cls, idx, session, refs, msgid, with_atts=False, cid=None, ephemeral=False): trees = [m.evaluate_pgp(m.get_message_tree(), decrypt=True) for m in refs] ref_subjs = [t['headers_lc']['subject'] for t in trees] msg_bodies = [] msg_atts = [] for t in trees: # FIXME: Templates/settings for how we quote forwards? text = '-------- Original Message --------\n' for h in ('Date', 'Subject', 'From', 'To'): v = t['headers_lc'].get(h.lower(), None) if v: text += '%s: %s\n' % (h, v) text += '\n' text += ''.join([p['data'] for p in t['text_parts'] if p['type'] in cls._TEXT_PARTTYPES]) msg_bodies.append(text) if with_atts: for att in t['attachments']: if att['mimetype'] not in cls._ATT_MIMETYPES: msg_atts.append(att['part']) if not ephemeral: local_id, lmbox = session.config.open_local_mailbox(session) else: local_id, lmbox = -1, None fmt = 'forward-att-%s-%s' if msg_atts else 'forward-%s-%s' ephemeral = [fmt % (msgid[1:-1].replace('@', '_'), refs[0].msg_mid())] if cid: # FIXME: Instead, we should use placeholders in the template # and insert the quoted bits in the right place (or # nowhere if the template doesn't want them). msg_bodies[:0] = [cls._get_canned(idx, cid)] email = Email.Create(idx, local_id, lmbox, msg_text='\n\n'.join(msg_bodies), msg_subject=cls.prefix_subject( ref_subjs[-1], 'Fwd:', cls._FW_REGEXP), msg_id=msgid, msg_atts=msg_atts, save=(not ephemeral), ephemeral_mid=ephemeral and ephemeral[0]) return email, ephemeral def command(self): session, config, idx = self.session, self.session.config, self._idx() with_atts = False ephemeral = False args = list(self.args) if not args: args = ["=%s" % x for x in self.data.get('mid', [])] ephemeral = truthy((self.data.get('ephemeral') or [False])[0]) with_atts = truthy((self.data.get('atts') or [False])[0]) else: while args: if args[0].lower() == 'att': with_atts = args.pop(0) or True elif args[0].lower() == 'ephemeral': ephemeral = args.pop(0) or True else: break # Make sure GET does not change backend state if self.data.get('_method', 'POST') != 'POST': ephemeral = True if ephemeral and with_atts: raise UsageError(_('Sorry, ephemeral messages cannot have ' 'attachments at this time.')) refs = [Email(idx, i) for i in self._choose_messages(args)] if refs: cid = self.data.get('cid', [None])[0] email, ephemeral = self.CreateForward(idx, session, refs, self._new_msgid(), with_atts=with_atts, cid=cid, ephemeral=ephemeral) if not ephemeral: self._track_action('fwded', refs) self._tag_blank([email]) return self._edit_messages([email], ephemeral=ephemeral) else: return self._error(_('No message found')) class Attach(CompositionCommand): """Attach a file to a message""" SYNOPSIS = ('a', 'attach', 'message/attach', ' []') ORDER = ('Composing', 2) WITH_CONTEXT = (GLOBAL_EDITING_LOCK, ) HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_QUERY_VARS = {} HTTP_POST_VARS = { 'mid': 'metadata-ID', 'name': '(ignored)', 'file-data': 'file data' } def command(self, emails=None): session, idx = self.session, self._idx() args = list(self.args) files = [] filedata = {} if 'file-data' in self.data: count = 0 for fd in self.data['file-data']: fn = (hasattr(fd, 'filename') and fd.filename or 'attach-%d.dat' % count) filedata[fn] = fd files.append(fn) count += 1 else: if args: fb = security.forbid_command(self, security.CC_ACCESS_FILESYSTEM) if fb: return self._error(fb) while os.path.exists(args[-1]): files.append(args.pop(-1)) if not files: return self._error(_('No files found')) if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) emails = [self._actualize_ephemeral(i) for i in self._choose_messages(args, allow_ephemeral=True)] if not emails: return self._error(_('No messages selected')) updated = [] errors = [] def err(msg): errors.append(msg) session.ui.error(msg) for email in emails: subject = email.get_msg_info(MailIndex.MSG_SUBJECT) try: email.add_attachments(session, files, filedata=filedata) updated.append(email) except KeyboardInterrupt: raise except NotEditableError: err(_('Read-only message: %s') % subject) except: err(_('Error attaching to %s') % subject) self._ignore_exception() file_list = ', '.join(f.decode('utf-8') for f in files) if errors: self.message = _('Attached %s to %d messages, failed %d' ) % (file_list, len(updated), len(errors)) else: self.message = _('Attached %s to %d messages' ) % (file_list, len(updated)) if updated: self._background_save(index=True) session.ui.notify(self.message) return self._return_search_results(self.message, updated, expand=updated, error=errors) class UnAttach(CompositionCommand): """Remove an attachment from a message""" SYNOPSIS = (None, 'unattach', 'message/unattach', ' ') ORDER = ('Composing', 2) WITH_CONTEXT = (GLOBAL_EDITING_LOCK, ) HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_QUERY_VARS = {} HTTP_POST_VARS = { 'mid': 'metadata-ID', 'att': 'Attachment IDs or filename' } def command(self, emails=None): session, idx = self.session, self._idx() args = list(self.args) atts = [] if '--' in args: atts = args[args.index('--') + 1:] args = args[:args.index('--')] elif args: atts = [args.pop(-1)] atts.extend(self.data.get('att', [])) if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) emails = [self._actualize_ephemeral(i) for i in self._choose_messages(args, allow_ephemeral=True)] if not emails: return self._error(_('No messages selected')) updated = [] errors = [] def err(msg): errors.append(msg) session.ui.error(msg) for email in emails: subject = email.get_msg_info(MailIndex.MSG_SUBJECT) try: email.remove_attachments(session, *atts) updated.append(email) except KeyboardInterrupt: raise except NotEditableError: err(_('Read-only message: %s') % subject) except: err(_('Error removing from %s') % subject) self._ignore_exception() if errors: self.message = _('Removed %s from %d messages, failed %d' ) % (', '.join(atts), len(updated), len(errors)) else: self.message = _('Removed %s from %d messages' ) % (', '.join(atts), len(updated)) if updated: self._background_save(index=True) session.ui.notify(self.message) return self._return_search_results(self.message, updated, expand=updated, error=errors) class Sendit(CompositionCommand): """Mail/bounce a message (to someone)""" SYNOPSIS = (None, 'bounce', 'message/send', ' []') ORDER = ('Composing', 5) HTTP_CALLABLE = ('POST', ) HTTP_QUERY_VARS = {} HTTP_POST_VARS = { 'mid': 'metadata-ID', 'to': 'recipients', 'from': 'sender e-mail' } # We set our events' source class explicitly, so subclasses don't # accidentally create orphaned mail tracking events. EVENT_SOURCE = 'mailpile.plugins.compose.Sendit' def command(self, emails=None): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) bounce_to = [] while args and '@' in args[-1]: bounce_to.append(args.pop(-1)) for rcpt in (self.data.get('to', []) + self.data.get('cc', []) + self.data.get('bcc', [])): bounce_to.extend(AddressHeaderParser(rcpt).addresses_list()) sender = self.data.get('from', [None])[0] if not sender and bounce_to: sender = idx.config.get_profile().get('email', None) if not emails: args.extend(['=%s' % mid for mid in self.data.get('mid', [])]) emails = [self._actualize_ephemeral(i) for i in self._choose_messages(args, allow_ephemeral=True)] # First make sure the draft tags are all gone, so other edits either # fail or complete while we wait for the lock. with GLOBAL_EDITING_LOCK: self._tag_drafts(emails, untag=True) self._tag_blank(emails, untag=True) # Process one at a time so we don't eat too much memory sent = [] missing_keys = [] locked_keys = [] for email in emails: events = [] try: msg_mid = email.get_msg_info(idx.MSG_MID) # This is a unique sending-ID. This goes in the public (meant # for debugging help) section of the event-log, so we take # care to not reveal details about the message or recipients. msg_sid = sha1b64(email.get_msg_info(idx.MSG_ID), *sorted(bounce_to))[:8] # We load up any incomplete events for sending this message # to this set of recipients. If nothing is in flight, create # a new event for tracking this operation. events = list(config.event_log.incomplete( source=self.EVENT_SOURCE, data_mid=msg_mid, data_sid=msg_sid)) if not events: events.append(config.event_log.log( source=self.EVENT_SOURCE, flags=Event.RUNNING, message=_('Sending message'), data={'mid': msg_mid, 'sid': msg_sid})) SendMail(session, msg_mid, [PrepareMessage(config, email.get_msg(pgpmime=False), sender=sender, rcpts=(bounce_to or None), bounce=(True if bounce_to else False), events=events)]) for ev in events: ev.flags = Event.COMPLETE config.event_log.log_event(ev) sent.append(email) # Encryption related failures are fatal, don't retry except (KeyLookupError, EncryptionFailureError, SignatureFailureError) as exc: message = unicode(exc) session.ui.warning(message) if hasattr(exc, 'missing_keys'): missing_keys.extend(exc.missing) if hasattr(exc, 'from_key'): # FIXME: We assume signature failures happen because # the key is locked. Are there any other reasons? locked_keys.append(exc.from_key) for ev in events: ev.flags = Event.COMPLETE ev.message = message config.event_log.log_event(ev) self._ignore_exception() # FIXME: Also fatal, when the SMTP server REJECTS the mail except: # We want to try that again! to = email.get_msg(pgpmime=False).get('x-mp-internal-rcpts', '').split(',')[0] if to: message = _('Could not send mail to %s') % to else: message = _('Could not send mail') for ev in events: ev.flags = Event.INCOMPLETE ev.message = message config.event_log.log_event(ev) session.ui.error(message) self._ignore_exception() if 'compose' in config.sys.debug: sys.stderr.write(('compose/Sendit: Send %s to %s (sent: %s)\n' ) % (len(emails), bounce_to or '(header folks)', sent)) if missing_keys: self.error_info['missing_keys'] = missing_keys if locked_keys: self.error_info['locked_keys'] = locked_keys if sent: self._tag_sent(sent) self._tag_outbox(sent, untag=True) for email in sent: email.reset_caches() idx.index_email(self.session, email) self._background_save(index=True) return self._return_search_results( _('Sent %d messages') % len(sent), sent, sent=sent) else: return self._error(_('Nothing was sent')) class Update(CompositionCommand): """Update message from a file or HTTP upload.""" SYNOPSIS = (None, 'update', 'message/update', ' <') ORDER = ('Composing', 1) WITH_CONTEXT = (GLOBAL_EDITING_LOCK, ) HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_POST_VARS = dict_merge(CompositionCommand.UPDATE_STRING_DATA, Attach.HTTP_POST_VARS) def command(self, create=True, outbox=False): session, config, idx = self.session, self.session.config, self._idx() email_updates = self._get_email_updates(idx, create=create, noneok=outbox) if not email_updates: return self._error(_('Nothing to do!')) try: if (self.data.get('file-data') or [''])[0]: if not Attach(session, data=self.data).command(emails=emails): return self._error(_('Failed to attach files')) for email, update_string in email_updates: if not email: return self._error(_('Cannot find message')) break email.update_from_string(session, update_string, final=outbox) emails = [e for e, u in email_updates] message = _('%d message(s) updated') % len(email_updates) self._tag_blank(emails, untag=True) self._tag_drafts(emails, untag=outbox) self._tag_outbox(emails, untag=(not outbox)) if outbox: self._create_contacts(emails) self._background_save(index=True) return self._return_search_results(message, emails, sent=emails) else: return self._edit_messages(emails, new=False, tag=False) except KeyLookupError as kle: return self._error(_('Missing encryption keys'), info={'missing_keys': kle.missing}) except EncryptionFailureError as efe: # This should never happen, should have been prevented at key # lookup! return self._error(_('Could not encrypt message'), info={'to_keys': efe.to_keys}) except SignatureFailureError as sfe: # FIXME: We assume signature failures happen because # the key is locked. Are there any other reasons? return self._error(_('Could not sign message'), info={'locked_keys': [sfe.from_key]}) class UpdateAndSendit(Update): """Update message from an HTTP upload and move to outbox.""" SYNOPSIS = ('m', 'mail', 'message/update/send', None) def command(self, create=True, outbox=True): return Update.command(self, create=create, outbox=outbox) class UnThread(CompositionCommand): """Remove a message from a thread.""" SYNOPSIS = (None, 'unthread', 'message/unthread', None) HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = { 'mid': 'message-id'} HTTP_POST_VARS = { 'subject': 'Update the metadata subject as well'} def command(self): session, config, idx = self.session, self.session.config, self._idx() args = list(self.args) # On the CLI, anything after -- is the new metadata subject. if '--' in args: subject = ' '.join(args[(args.index('--')+1):]) args = args[:args.index('--')] else: subject = self.data.get('subject', [None])[0] # Message IDs can come from post data for mid in self.data.get('mid', []): args.append('=%s' % mid) emails = [self._actualize_ephemeral(i) for i in self._choose_messages(args, allow_ephemeral=True)] if emails: if self.data.get('_method', 'POST') == 'POST': for email in emails: idx.unthread_message(email.msg_mid(), new_subject=subject) self._background_save(index=True) return self._return_search_results( _('Unthreaded %d messages') % len(emails), emails) else: return self._return_search_results( _('Unthread %d messages') % len(emails), emails) else: return self._error(_('Nothing to do!')) class EmptyOutbox(Sendit): """Try to empty the outbox.""" SYNOPSIS = (None, 'sendmail', None, None) IS_USER_ACTIVITY = False @classmethod def sendmail(cls, session): cls(session).run() def command(self): cfg, idx = self.session.config, self.session.config.index if not idx: return self._error(_('The index is not ready yet')) # Collect a list of messages from the outbox messages = [] for tag in cfg.get_tags(type='outbox'): search = ['in:%s' % tag._key] for msg_idx_pos in idx.search(self.session, search, order='flat-index').as_set(): messages.append('=%s' % b36(msg_idx_pos)) # Messages no longer in the outbox get their events canceled... if cfg.event_log: events = cfg.event_log.incomplete(source='.plugins.compose.Sendit') for ev in events: if ('mid' in ev.data and ('=%s' % ev.data['mid']) not in messages): ev.flags = ev.COMPLETE ev.message = _('Sending cancelled.') cfg.event_log.log_event(ev) # Send all the mail! if messages: self.args = tuple(set(messages)) return Sendit.command(self) else: return self._success(_('The outbox is empty')) _plugins.register_config_variables('prefs', { 'empty_outbox_interval': [_('Delay between attempts to send mail'), int, 90] }) _plugins.register_slow_periodic_job('sendmail', 'prefs.empty_outbox_interval', EmptyOutbox.sendmail) _plugins.register_commands(Compose, Reply, Forward, # Create Draft, Update, Attach, UnAttach, # Manipulate UnThread, # ... Sendit, UpdateAndSendit, # Send EmptyOutbox) # ... ================================================ FILE: mailpile/plugins/contacts.py ================================================ import os import random import time from email import encoders import mailpile.config.defaults import mailpile.security as security from mailpile.crypto.gpgi import GnuPG from mailpile.crypto.gpgi import GnuPGBaseKeyGenerator, GnuPGKeyGenerator from mailpile.crypto.autocrypt import generate_autocrypt_setup_code from mailpile.plugins import EmailTransform, PluginManager from mailpile.commands import Command, Action from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailutils.addresses import AddressHeaderParser from mailpile.mailutils.emails import Email, ExtractEmails, ExtractEmailAndName from mailpile.security import SecurePassphraseStorage from mailpile.vcard import VCardLine, VCardStore, MailpileVCard, AddressInfo, GLOBAL_VCARD_LOCK from mailpile.util import * _plugins = PluginManager(builtin=__file__) ##[ VCards ]######################################## class VCardCommand(Command): VCARD = "vcard" IS_USER_ACTIVITY = True WITH_CONTEXT = (GLOBAL_VCARD_LOCK,) class CommandResult(Command.CommandResult): IGNORE = ('line_id', 'pid', 'x-rank') def as_text(self): try: return self._as_text() except (KeyError, ValueError, IndexError, TypeError): return '' def _as_text(self): if isinstance(self.result, dict): co = self.command_obj if co.VCARD in self.result: return self._vcards_as_text([self.result[co.VCARD]]) if co.VCARD + 's' in self.result: return self._vcards_as_text(self.result[co.VCARD + 's']) return Command.CommandResult.as_text(self) def _vcards_as_text(self, result): lines = [] b64re = re.compile('base64,.*$') for card in result: if isinstance(card, list): for line in card: key = line.name data = re.sub(b64re, _('(BASE64 ENCODED DATA)'), unicode(line[key])) attrs = ', '.join([('%s=%s' % (k, v)) for k, v in line.attrs if k not in ('pid',)]) if attrs: attrs = ' (%s)' % attrs lines.append('%3.3s %-5.5s %s: %s%s' % (line.line_id, line.get('pid', ''), key, data, attrs)) lines.append('') else: emails = [k['email'] for k in card['email']] photos = [k['photo'] for k in card.get('photo', [])] lines.append('%s %-32.32s %s' % (photos and ':)' or ' ', card['fn'] + (' (%s)' % card['note'] if card.get('note') else ''), ', '.join(emails))) for key in [k['key'].split(',')[-1] for k in card.get('key', [])]: lines.append(' %-32.32s key:%s' % ('', key)) return '\n'.join(lines) def _form_defaults(self): return {'form': self.HTTP_POST_VARS} def _make_new_vcard(self, handle, name, note, kind): l = [VCardLine(name='fn', value=name), VCardLine(name='kind', value=kind)] if note: l.append(VCardLine(name='note', value=note)) if kind in VCardStore.KINDS_PEOPLE: return MailpileVCard(VCardLine(name='email', value=handle, type='pref'), *l, config=self.session.config) else: return MailpileVCard(VCardLine(name='nickname', value=handle), *l, config=self.session.config) def _valid_vcard_handle(self, vc_handle): return (vc_handle and '@' in vc_handle[1:]) def _pre_delete_vcard(self, vcard): pass def _vcard_list(self, vcards, mode='mpCard', info=None, simplify=False): info = info or {} if mode == 'lines': data = [x.as_lines() for x in vcards if x] else: data = [x.as_mpCard() for x in vcards if x] # Generate some helpful indexes for finding stuff by_email = {} by_rid = {} for count, vc in enumerate(vcards): by_rid[vc.random_uid] = count by_email[vc.email] = count for count, vc in enumerate(vcards): for vcl in vc.get_all('EMAIL'): if vcl.value not in by_email: by_email[vcl.value] = count # Simplify lists when there is only one element? if simplify and len(data) == 1: data = data[0] whatsit = self.VCARD else: whatsit = self.VCARD + 's' info.update({ whatsit: data, "emails": by_email, "rids": by_rid, "count": len(vcards) }) return info class VCard(VCardCommand): """Display a single vcard""" SYNOPSIS = (None, 'vcards/view', None, '') ORDER = ('Internals', 6) KIND = '' def command(self, save=True): self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config vcards = [] for email in self.args: vcard = config.vcards.get_vcard(email) if vcard: vcards.append(vcard) else: session.ui.warning('No such %s: %s' % (self.VCARD, email)) return self._success(_('Found %d results') % len(vcards), result=self._vcard_list(vcards, simplify=True)) class AddVCard(VCardCommand): """Add one or more vcards""" SYNOPSIS = (None, 'vcards/add', None, '[all] OR = ') ORDER = ('Internals', 6) KIND = '' HTTP_CALLABLE = ('POST', 'PUT', 'GET') HTTP_POST_VARS = { 'email': 'E-mail address', 'name': 'Contact name', 'note': 'Note about contact', 'mid': 'Message ID' } COMMAND_SECURITY = security.CC_CHANGE_CONTACTS IGNORED_EMAILS_AND_DOMAINS = ( 'reply.airbnb.com', 'notifications@github.com' ) def _add_from_messages(self, args, add_recipients): pairs, idx = [], self._idx() for email in [Email(idx, i) for i in self._choose_messages(args)]: msg_info = email.get_msg_info() pairs.append(ExtractEmailAndName(msg_info[idx.MSG_FROM])) if add_recipients: people = (idx.expand_to_list(msg_info) + idx.expand_to_list(msg_info, field=idx.MSG_CC)) for e in people: pair = ExtractEmailAndName(e) domain = pair[0].split('@')[-1] if (pair[0] not in self.IGNORED_EMAILS_AND_DOMAINS and domain not in self.IGNORED_EMAILS_AND_DOMAINS and 'noreply' not in pair[0]): pairs.append(pair) return [(p1, p2, '') for p1, p2 in pairs] def _sanity_check(self, kind, vcard): pass def _before_vcard_create(self, kind, triplets): return {} def _after_vcard_create(self, kind, vcard, state): pass def command(self, recipients=False, quietly=False, internal=False): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config args = list(self.args) if self.data.get('_method', 'not-http').upper() == 'GET': return self._success(_('Add contacts here!'), self._form_defaults()) if (len(args) > 2 and args[1] == '=' and self._valid_vcard_handle(args[0])): handle = args[0] name = [] note = [] inname = True for v in args[2:]: if v.startswith('('): inname = False v = v[1:] if v.endswith(')'): v = v[:-1] if inname: name.append(v) else: note.append(v) triplets = [(args[0], ' '.join(name), ' '.join(note))] elif self.data: if self.data.get('email'): emails = self.data["email"] names = self.data["name"][:] names.extend(['' for i in range(len(names), len(emails))]) notes = self.data.get("note", [])[:] notes.extend(['' for i in range(len(notes), len(emails))]) triplets = zip(emails, names, notes) elif self.data.get('mid'): mids = self.data.get('mid') triplets = self._add_from_messages( ['=%s' % mid.replace('=', '') for mid in mids]) else: if args and args[0] == 'all': recipients = args.pop(0) and True triplets = self._add_from_messages(args, recipients) if triplets: vcards = [] kind = self.KIND if not internal else 'internal' self._sanity_check(kind, triplets) state = self._before_vcard_create(kind, triplets) for handle, name, note in triplets: vcard = config.vcards.get(handle.lower()) if vcard: if not quietly: session.ui.warning('Already exists: %s' % handle) if kind != 'profile' and vcard.kind != 'internal': continue if vcard and vcard.kind == 'internal': config.vcards.deindex_vcard(vcard) vcard.email = handle.lower() vcard.name = name vcard.note = note vcard.kind = kind else: vcard = self._make_new_vcard(handle.lower(), name, note, kind) self._after_vcard_create(kind, vcard, state) config.vcards.add_vcards(vcard) vcards.append(vcard) if state.get('save_config', False): self._background_save(config=True) else: return self._error('Nothing to do!') return self._success(_('Added %d contacts') % len(vcards), result=self._vcard_list(vcards, simplify=True)) class RemoveVCard(VCardCommand): """Delete vcards""" SYNOPSIS = (None, 'vcards/remove', None, '') ORDER = ('Internals', 6) KIND = '' HTTP_CALLABLE = ('POST', 'DELETE') HTTP_POST_VARS = { 'email': 'delete by e-mail', 'rid': 'delete by x-mailpile-rid' } COMMAND_SECURITY = security.CC_CHANGE_CONTACTS def command(self): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config removed = [] for handle in (list(self.args) + self.data.get('email', []) + self.data.get('rid', [])): vcard = config.vcards.get_vcard(handle) if vcard: self._pre_delete_vcard(vcard) config.vcards.del_vcards(vcard) removed.append(handle) else: session.ui.error(_('No such contact: %s') % handle) if removed: return self._success(_('Removed contacts: %s') % ', '.join(removed)) else: return self._error(_('No contacts found')) class VCardAddLines(VCardCommand): """Add a lines to a VCard""" SYNOPSIS = (None, 'vcards/addlines', 'vcards/addlines', ' <[[]=]line> ...') ORDER = ('Internals', 6) KIND = '' HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_POST_VARS = { 'email': 'update by e-mail', 'rid': 'update by x-mailpile-rid', 'name': 'Line name', 'value': 'Line value', 'replace': 'int=replace line by number', 'replace_all': 'Boolean: replace all lines, or not', 'client': 'Source of this change' } COMMAND_SECURITY = security.CC_CHANGE_CONTACTS DEFAULT_REPLACE_ALL = False def _get_vcard(self, handle): return self.session.config.vcards.get_vcard(handle) def command(self): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config if self.args: handle, lines = self.args[0], self.args[1:] else: handle = self.data.get('rid', self.data.get('email', [None]))[0] if not handle: raise ValueError('Must set rid or email to choose VCard') name, value, replace, replace_all = (self.data.get(n, [None])[0] for n in ('name', 'value', 'replace', 'replace_all')) if not name or not value or ':' in name or '=' in name: raise ValueError('Must send a line name and line data') value = '%s:%s' % (name, value) if replace: value = '%d=%s' % (replace, value) elif truthy(replace_all, default=self.DEFAULT_REPLACE_ALL): value = '=' + value lines = [value] vcard = self._get_vcard(handle) if not vcard: return self._error('%s not found: %s' % (self.VCARD, handle)) config.vcards.deindex_vcard(vcard) client = self.data.get('client', [vcard.USER_CLIENT])[0] try: for l in lines: lname = l.split(':', 1)[0].lower() if lname[0] == '=': l = l[1:].strip() lname = lname[1:] removing = [ex._line_id for ex in vcard.get_all(lname)] elif lname in MailpileVCard.MPCARD_SINGLETONS: removing = [ex._line_id for ex in vcard.get_all(lname)] else: removing = [] if '=' in l[:5]: ln, l = l.split('=', 1) vcard.set_line(int(ln.strip()), VCardLine(l.strip()), client=client) else: if removing: vcard.remove(*removing) vcard.add(VCardLine(l), client=client) vcard.save() return self._success(_("Added %d lines") % len(lines), result=self._vcard_list([vcard], simplify=True, info={ 'updated': handle, 'added': len(lines) })) except KeyboardInterrupt: raise except: config.vcards.index_vcard(vcard) self._ignore_exception() return self._error(_('Error adding lines to %s') % handle) finally: config.vcards.index_vcard(vcard) class VCardSet(VCardAddLines): """Add a lines to a VCard, ensuring VCard exists""" SYNOPSIS = (None, 'vcards/set', 'vcards/set', ' <[[]=]line> ...') HTTP_POST_VARS = dict_merge(VCardAddLines.HTTP_POST_VARS, { 'fn': 'Name on card' }) DEFAULT_REPLACE_ALL = True def _get_vcard(self, handle): vcard = self.session.config.vcards.get_vcard(handle) if not vcard: vcard = self._make_new_vcard(handle, self.data.get('fn', [handle])[0], None, self.KIND or 'individual') self.session.config.vcards.add_vcards(vcard) return vcard class VCardRemoveLines(VCardCommand): """Remove lines from a VCard""" SYNOPSIS = (None, 'vcards/rmlines', 'vcards/rmlines', ' ') ORDER = ('Internals', 6) KIND = '' HTTP_CALLABLE = ('POST', 'UPDATE') COMMAND_SECURITY = security.CC_CHANGE_CONTACTS HTTP_POST_VARS = { 'email': 'update by e-mail', 'rid': 'update by x-mailpile-rid', 'name': 'Line names', 'line_id': 'Line IDs'} def command(self): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config if self.args: handle, names, line_ids = self.args[0], [], [] for arg in self.args[1:]: try: line_ids.append('%d' % int(arg)) except ValueError: names.append(arg) else: handle = self.data.get('rid', self.data.get('email', [None]))[0] if not handle: raise ValueError('Must set rid or email to choose VCard') names = self.data.get('name', []) line_ids = self.data.get('line_id', []) if not (names or line_ids): raise ValueError('Must send a line name or line ID') vcard = config.vcards.get_vcard(handle) if not vcard: return self._error('%s not found: %s' % (self.VCARD, handle)) config.vcards.deindex_vcard(vcard) removed = 0 try: for lname in names: line_ids.extend(ex._line_id for ex in vcard.get_all(lname)) removed += vcard.remove(*[int(li) for li in line_ids]) vcard.save() return self._success(_("Removed %d lines") % removed, result=self._vcard_list([vcard], simplify=True, info={ 'updated': handle, 'removed': removed })) except KeyboardInterrupt: raise except: config.vcards.index_vcard(vcard) self._ignore_exception() return self._error(_('Error removing lines from %s') % handle) finally: config.vcards.index_vcard(vcard) class ListVCards(VCardCommand): """Find vcards""" SYNOPSIS = (None, 'vcards', None, '[--lines] []') ORDER = ('Internals', 6) KIND = '' HTTP_QUERY_VARS = { 'q': 'search terms', 'format': 'lines or mpCard (default)', 'count': 'how many to display (default=40)', 'offset': 'skip how many in the display (default=0)', } HTTP_CALLABLE = ('GET') def _augment_list_info(self, info): return info def command(self): session, config = self.session, self.session.config kinds = self.KIND and [self.KIND] or None args = list(self.args) if 'format' in self.data: fmt = self.data['format'][0] elif args and args[0] == '--lines': args.pop(0) fmt = 'lines' else: fmt = 'mpCard' if 'q' in self.data: terms = self.data['q'] else: terms = args if 'count' in self.data: count = int(self.data['count'][0]) else: count = 120 if 'offset' in self.data: offset = int(self.data['offset'][0]) else: offset = 0 # If we're loading, stall a bit but then report current state loading, loaded = config.vcards.loading, config.vcards.loaded if loading: time.sleep(2) loading, loaded = config.vcards.loading, config.vcards.loaded vcards = config.vcards.find_vcards(terms, kinds=kinds) total = len(vcards) vcards = vcards[offset:offset + count] info = self._augment_list_info({ 'terms': args, 'offset': offset, 'count': min(count, total), 'total': total, 'start': offset, 'end': offset + min(count, total - offset), 'loading': loading, 'loaded': loaded}) return self._success( _("Listed %d/%d results") % (min(total, count), total), result=self._vcard_list(vcards, mode=fmt, info=info)) def ContactVCard(parent): """A factory for generating contact commands""" synopsis = [(t and t.replace('vcard', 'contact') or t) for t in parent.SYNOPSIS] synopsis[2] = synopsis[1] class ContactVCardCommand(parent): SYNOPSIS = tuple(synopsis) KIND = 'individual' ORDER = ('Tagging', 3) VCARD = "contact" return ContactVCardCommand class Contact(ContactVCard(VCard)): """View contacts""" SYNOPSIS = (None, 'contacts/view', 'contacts/view', '[]') def command(self, save=True): contact = VCard.command(self, save) # Tee-hee, monkeypatching results. contact["sent_messages"] = 0 contact["received_messages"] = 0 contact["last_contact_from"] = 10000000000000 contact["last_contact_to"] = 10000000000000 for email in contact["contact"]["email"]: s = Action(self.session, "search", ["in:Sent", "to:%s" % (email["email"])]).as_dict() contact["sent_messages"] += s["result"]["stats"]["total"] for mid in s["result"]["thread_ids"]: msg = s["result"]["data"]["metadata"][mid] if msg["timestamp"] < contact["last_contact_to"]: contact["last_contact_to"] = msg["timestamp"] contact["last_contact_to_msg_url"] = msg["urls"]["thread"] s = Action(self.session, "search", ["from:%s" % (email["email"])]).as_dict() contact["received_messages"] += s["result"]["stats"]["total"] for mid in s["result"]["thread_ids"]: msg = s["result"]["data"]["metadata"][mid] if msg["timestamp"] < contact["last_contact_from"]: contact["last_contact_from"] = msg["timestamp"] contact["last_contact_from_msg_url" ] = msg["urls"]["thread"] if contact["last_contact_to"] == 10000000000000: contact["last_contact_to"] = False contact["last_contact_to_msg_url"] = "" if contact["last_contact_from"] == 10000000000000: contact["last_contact_from"] = False contact["last_contact_from_msg_url"] = "" return contact class AddContact(ContactVCard(AddVCard)): """Add contacts""" class RemoveContact(ContactVCard(RemoveVCard)): """Remove a contact""" class ListContacts(ContactVCard(ListVCards)): SYNOPSIS = (None, 'contacts', 'contacts', '[--lines] []') """Find contacts""" class ContactSet(ContactVCard(VCardSet)): """Set contact lines, ensuring contact exists""" class ContactImport(Command): """Import contacts""" SYNOPSIS = (None, 'contacts/import', 'contacts/import', '[]') ORDER = ('Internals', 6) HTTP_CALLABLE = ('GET', ) COMMAND_SECURITY = security.CC_CHANGE_CONTACTS def command(self, format, terms=None, **kwargs): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config if not format in PluginManager.CONTACT_IMPORTERS.keys(): session.ui.error("No such import format") return False importer = PluginManager.CONTACT_IMPORTERS[format] if not all([x in kwargs.keys() for x in importer.required_parameters]): session.ui.error( _("Required parameter missing. Required parameters " "are: %s") % ", ".join(importer.required_parameters)) return False allparams = importer.required_parameters + importer.optional_parameters if not all([x in allparams for x in kwargs.keys()]): session.ui.error( _("Unknown parameter passed to importer. " "Provided %s; but known parameters are: %s" ) % (", ".join(kwargs), ", ".join(allparams))) return False imp = importer(kwargs) if terms: contacts = imp.filter_contacts(terms) else: contacts = imp.get_contacts() for importedcontact in contacts: # Check if contact exists. If yes, then update. Else create. pass class ContactImporters(Command): """Return a list of contact importers""" SYNOPSIS = (None, 'contacts/importers', 'contacts/importers', '') ORDER = ('Internals', 6) HTTP_CALLABLE = ('GET', ) def command(self): res = [] for iname, importer in CONTACT_IMPORTERS.iteritems(): r = {} r["short_name"] = iname r["format_name"] = importer.format_name r["format_description"] = importer.format_description r["optional_parameters"] = importer.optional_parameters r["required_parameters"] = importer.required_parameters res.append(r) return res class AddressSearch(VCardCommand): """Find addresses (in contacts or mail index)""" SYNOPSIS = (None, 'search/address', 'search/address', '[]') ORDER = ('Searching', 6) HTTP_QUERY_VARS = { 'q': 'search terms', 'count': 'number of results', 'offset': 'offset results', 'ms': 'deadline in ms' } def _boost_rank(self, boost, term, *matches): boost = 0.0 for match in matches: match = match.lower() if term in match: if match.startswith(term): boost += boost * boost * (float(len(term)) / len(match)) else: boost += boost * (float(len(term)) / len(match)) return int(boost) def _vcard_addresses(self, cfg, terms, ignored_count, deadline): addresses = {} for vcard in cfg.vcards.find_vcards(terms, kinds=VCardStore.KINDS_PEOPLE): fn = vcard.get('fn') for email_vcl in vcard.get_all('email'): info = addresses.get(email_vcl.value) or {} info.update(AddressInfo(email_vcl.value, fn.value, vcard=vcard)) info['rank'] = min(15, info.get('rank', 15)) addresses[email_vcl.value] = info for term in terms: info['rank'] += self._boost_rank(5, term, fn.value, email_vcl.value) if len(addresses) and time.time() > deadline: break return addresses.values() def _index_addresses(self, cfg, terms, vcard_addresses, count, deadline): existing = dict([(k['address'].lower(), k) for k in vcard_addresses]) index = self._idx() # Figure out which tags are invisible so we can skip messages marked # with those tags. invisible = set([t._key for t in cfg.get_tags(flag_hides=True)]) matches = {} addresses = [] # 1st, search the social graph for matches, give low priority. for frm in index.EMAILS: frm_lower = frm.lower() match = True for term in terms: if term not in frm_lower: match = False break if match: matches[frm] = matches.get(frm, 0) + 3 if len(matches) > (count * 10): break elif len(matches) and time.time() > deadline: break # 2nd, go through at most the last 5000 messages in the index and # search for matching senders or recipients, give medium priority. # Note: This is more CPU intensive, so we do this last. if len(matches) < (count * 5): for msg_idx in xrange(max(0, len(index.INDEX)-5000), len(index.INDEX)): msg_info = index.get_msg_at_idx_pos(msg_idx) tags = set(msg_info[index.MSG_TAGS].split(',')) match = not (tags & invisible) if match: frm = msg_info[index.MSG_FROM] search = (frm + ' ' + msg_info[index.MSG_SUBJECT]).lower() for term in terms: if term not in search: match = False break if match: matches[frm] = matches.get(frm, 0) + 1 if len(matches) > (count * 5): break if len(matches) and time.time() > deadline: break # Assign info & scores! for frm in matches: email, fn = ExtractEmailAndName(frm) boost = min(10, matches[frm]) for term in terms: boost += self._boost_rank(4, term, fn, email) if not email or '@' not in email: # FIXME: This may not be the right thing for alternate # message transports. pass elif email.lower() in existing: existing[email.lower()]['rank'] += boost else: info = AddressInfo(email, fn) info['rank'] = info.get('rank', 0) + boost existing[email.lower()] = info addresses.append(info) return addresses def command(self): session, config = self.session, self.session.config count = int(self.data.get('count', 10)) offset = int(self.data.get('offset', 0)) deadline = time.time() + float(self.data.get('ms', 150)) / 1000.0 terms = [] for q in self.data.get('q', []): terms.extend(q.lower().split()) for a in self.args: terms.extend(a.lower().split()) self.session.ui.mark('Searching VCards') vcard_addrs = self._vcard_addresses(config, terms, count, deadline) self.session.ui.mark('Searching Metadata') index_addrs = self._index_addresses(config, terms, vcard_addrs, count, deadline) self.session.ui.mark('Sorting') addresses = vcard_addrs + index_addrs addresses.sort(key=lambda k: -k['rank']) total = len(addresses) return self._success(_('Searched for addresses'), result={ 'addresses': addresses[offset:min(offset+count, total)], 'displayed': min(count, total), 'total': total, 'offset': offset, 'count': count, 'start': offset, 'end': offset+count, }) def ProfileVCard(parent): """A factory for generating profile commands""" synopsis = [(t and t.replace('vcard', 'profile') or t) for t in parent.SYNOPSIS] synopsis[2] = synopsis[1] class ProfileVCardCommand(parent): SYNOPSIS = tuple(synopsis) KIND = 'profile' ORDER = ('Tagging', 3) VCARD = "profile" DEFAULT_KEYTYPE = 'RSA3072' def _default_signature(self): return _('Sent using Mailpile, Free Software from www.mailpile.is') def _augment_list_info(self, info): info['default_sig'] = self._default_signature() return info def _yn(self, val, default='no'): return truthy(self.data.get(val, [default])[0]) def _sendmail_command(self): # FIXME - figure out where sendmail is for reals return mailpile.config.defaults.DEFAULT_SENDMAIL def _sanity_check(self, kind, triplets): route_id = self.data.get('route_id', [None])[0] if (route_id or [k for k in self.data.keys() if k[:5] in ('route', 'smtp-', 'sourc', 'secur', 'local')]): if len(triplets) > 1 or kind != 'profile': raise ValueError('Can only configure detailed settings ' 'for one profile at a time') # FIXME: Check more important invariants and raise def _configure_sending_route(self, vcard, route_id): # Sending route route = self.session.config.routes.get(route_id) protocol = self.data.get('route-protocol', ['none'])[0] if protocol == 'none': try: del self.session.config.routes[route_id] except (KeyError, IndexError): pass vcard.route = '' return elif protocol == 'local': route.password = route.username = route.host = '' route.name = _("Local mail") route.command = self.data.get('route-command', [None] )[0] or self._sendmail_command() elif protocol in ('smtp', 'smtptls', 'smtpssl'): route.command = '' route.name = vcard.email for var in ('route-username', 'route-auth_type', 'route-host', 'route-port'): rvar = var.split('-', 1)[1] route[rvar] = self.data.get(var, [''])[0] if not self.data.get('route-username', [''])[0]: route['auth_type'] = '' if 'route-password' in self.data: route['password'] = self.data['route-password'][0] else: raise ValueError(_('Unhandled outgoing mail protocol: %s' ) % protocol) route.protocol = protocol vcard.route = route_id def _get_mail_spool(self): path = os.getenv('MAIL') or None user = os.getenv('USER') if user and not path: if os.path.exists('/var/spool/mail'): path = os.path.normpath('/var/spool/mail/%s' % user) if os.path.exists('/var/mail'): path = os.path.normpath('/var/mail/%s' % user) return path def _configure_mail_sources(self, vcard): config = self.session.config sources = [r[7:].rsplit('-', 1)[0] for r in self.data.keys() if r.startswith('source-') and r.endswith('-protocol')] for src_id in sources: prefix = 'source-%s-' % src_id protocol = self.data.get(prefix + 'protocol', ['none'])[0] def configure_source(source): source.host = '' source.username = '' source.enabled = self._yn(prefix + 'enabled') source.discovery.create_tag = True source.discovery.process_new = True if src_id not in vcard.sources(): vcard.add_source(source._key) return source def make_new_source(): # This little dance makes sure source is actually a # config section, not just an anonymous dict. if src_id not in config.sources: config.sources[src_id] = {} source = config.sources[src_id] source.profile = vcard.random_uid source.discovery.apply_tags = [vcard.tag] return configure_source(source) if protocol == 'none': pass elif protocol == 'local': source = configure_source(vcard.get_source_by_proto( 'local', create=src_id)) elif protocol == 'spool': path = self._get_mail_spool() if not path: raise ValueError(_('Mail spool not found')) if path in config.sys.mailbox.values(): raise ValueError(_('Already configured: %s') % path) else: mailbox_idx = config.sys.mailbox.append(path) source = configure_source(vcard.get_source_by_proto( 'local', create=src_id)) src_id = source._key # We need to communicate with the source below, # so we save config to trigger instanciation. self._background_save(config=True, wait=True) inbox = [t._key for t in config.get_tags(type='inbox')] local_copy = self._yn(prefix + 'copy-local') if self._yn(prefix + 'delete-source'): policy = 'move' else: policy = 'read' src_obj = config.mail_sources[src_id] src_obj.take_over_mailbox(mailbox_idx, policy=policy, create_local=local_copy, apply_tags=inbox, save=False) elif protocol in ('imap', 'imap_ssl', 'imap_tls', 'pop3', 'pop3_ssl'): source = make_new_source() # Discovery policy disco = source.discovery if self._yn(prefix + 'index-all-mail'): if self._yn(prefix + 'leave-on-server'): disco.policy = 'sync' else: disco.policy = 'move' disco.local_copy = True disco.paths = ['/'] else: disco.policy = 'ignore' disco.local_copy = False disco.paths = [] disco.guess_tags = True # Connection settings for rvar in ('protocol', 'auth_type', 'username', 'host', 'port'): source[rvar] = self.data.get(prefix + rvar, [''])[0] if (prefix + 'password') in self.data: source['password'] = self.data[prefix + 'password'][0] if (self._yn(prefix + 'force-starttls') and source.protocol == 'imap'): source.protocol = 'imap_tls' username = source.username if '@' not in username: username += '@%s' % source.host source.name = username # We need to communicate with the source below, # so we save config to trigger instanciation. self._background_save(config=True, wait=True) src_obj = config.mail_sources[src_id] else: raise ValueError(_('Unhandled incoming mail protocol: %s' ) % protocol) def _new_key_created(self, event, vcard_rid, passphrase): config = self.session.config fingerprint = self._key_generator.generated_key if fingerprint: with GLOBAL_VCARD_LOCK: vcard = vcard_rid and config.vcards.get_vcard(vcard_rid) if vcard: vcard.pgp_key = fingerprint vcard.save() event.message = _('The PGP key for %s is ready for use.' ) % vcard.email else: event.message = _('PGP key generation is complete') # Record the passphrase! config.secrets[fingerprint] = { 'password': passphrase, 'policy': 'protect'} # FIXME: Toggle something that indicates we need a backup ASAP. self._background_save(config=True) else: event.message = _('PGP key generation failed!') event.data['keygen_failed'] = True event.flags = event.COMPLETE event.data['keygen_finished'] = int(time.time()) config.event_log.log_event(event) def _create_new_key(self, vcard, keytype_arg): passphrase = generate_autocrypt_setup_code() random_uid = vcard.random_uid if keytype_arg[:3].upper() == 'RSA': keytype = GnuPGBaseKeyGenerator.KEYTYPE_RSA bits = int(keytype_arg[3:]) elif keytype_arg.upper() in ('ECC', 'ED25519', 'CURVE25519'): keytype = GnuPGBaseKeyGenerator.KEYTYPE_CURVE25519 bits = None else: raise ValueError('Unknown keytype: %s' % keytype_arg) key_args = { 'keytype': keytype, 'bits': bits, 'name': vcard.fn, 'email': vcard.email, 'passphrase': passphrase, 'comment': ''} event = Event(source=self, flags=Event.INCOMPLETE, data={'keygen_started': int(time.time()), 'profile_id': random_uid}, private_data=key_args) self._key_generator = GnuPGKeyGenerator( # FIXME: Passphrase handling is a problem here GnuPG(self.session.config, event=event), event=event, variables=dict_merge(GnuPGBaseKeyGenerator.VARIABLES, key_args), on_complete=(random_uid, lambda: self._new_key_created(event, random_uid, passphrase))) self._key_generator.start() self.session.config.event_log.log_event(event) def _configure_security(self, vcard): openpgp_key = self.data.get('security-pgp-key', [''])[0] if openpgp_key: if openpgp_key.startswith('!CREATE'): key_type = openpgp_key[8:] or self.DEFAULT_KEYTYPE self._create_new_key(vcard, key_type) else: vcard.pgp_key = openpgp_key # FIXME: Schedule a background sync job which edits # the key to add this Account as a UID, if it else: vcard.remove_all('key') # Set the following even if we don't have a key, so they don't # get lost if the user edits settings while a key is being # generated - or if they just deselect a key temporarily. # Encryption policy rules outg_auto = self._yn('security-best-effort-crypto') outg_ac11 = self._yn('security-autocrypt-crypto') outg_sig = self._yn('security-always-sign') outg_enc = self._yn('security-always-encrypt') if outg_ac11: vcard.crypto_policy = 'autocrypt' elif outg_enc and outg_sig: vcard.crypto_policy = 'openpgp-sign-encrypt' elif outg_sig: vcard.crypto_policy = 'openpgp-sign' elif outg_enc: vcard.crypto_policy = 'openpgp-encrypt' elif outg_auto: vcard.crypto_policy = 'best-effort' else: vcard.crypto_policy = 'none' # Crypto formatting rules pgp_autocrypt = outg_ac11 or self._yn('security-use-autocrypt') pgp_publish = self._yn('security-publish-to-keyserver') pgp_keys = self._yn('security-attach-keys') pgp_inline = self._yn('security-prefer-inline') pgp_pgpmime = self._yn('security-prefer-pgpmime') pgp_obscure_meta = self._yn('security-obscure-metadata') pgp_hdr_enc = self._yn('security-openpgp-header-encrypt') pgp_hdr_sig = self._yn('security-openpgp-header-sign') pgp_hdr_none = self._yn('security-openpgp-header-none') pgp_hdr_both = pgp_hdr_enc and pgp_hdr_sig if pgp_hdr_both: pgp_hdr_enc = pgp_hdr_sig = False if pgp_pgpmime and pgp_inline: pgp_pgpmime = pgp_inline = False vcard.crypto_format = ''.join([ 'openpgp_header:SE' if (pgp_hdr_both) else '', 'openpgp_header:S' if (pgp_hdr_sig) else '', 'openpgp_header:E' if (pgp_hdr_enc) else '', 'openpgp_header:N' if (pgp_hdr_none) else '', '+autocrypt' if (pgp_autocrypt) else '', '+send_keys' if (pgp_keys) else '', '+prefer_inline' if (pgp_inline) else '', '+pgpmime' if (pgp_pgpmime) else '', '+obscure_meta' if (pgp_obscure_meta) else '', '+publish' if (pgp_publish) else '' ]) return ProfileVCardCommand class Profile(ProfileVCard(VCard)): """View profile""" class AddProfile(ProfileVCard(AddVCard)): """Add profiles (Accounts)""" HTTP_POST_VARS = dict_merge(AddVCard.HTTP_POST_VARS, { 'route_id': 'Route ID for sending mail', 'signature': '.signature', 'route-*': 'Route settings', 'source-*': 'Source settings', 'security-*': 'Security settings' }) def _form_defaults(self): new_src_id = randomish_uid(); return dict_merge(AddVCard._form_defaults(self), { 'new_src_id': new_src_id, 'signature': self._default_signature(), 'route-protocol': 'none', 'route-auth_type': 'password', 'source-NEW-protocol': 'none', 'source-NEW-auth_type': 'password', 'source-NEW-leave-on-server': True, 'source-NEW-index-all-mail': True, 'source-NEW-force-starttls': False, 'source-NEW-copy-local': True, 'source-NEW-delete-source': False, 'security-best-effort-crypto': True, 'security-autocrypt-crypto': False, 'security-always-sign': False, 'security-always-encrypt': False, 'security-use-autocrypt': True, 'security-attach-keys': False, 'security-prefer-inline': False, 'security-prefer-pgpmime': False, 'security-obscure-metadata': False, 'security-openpgp-header-encrypt': False, 'security-openpgp-header-sign': True, 'security-openpgp-header-none': False, 'security-publish-to-keyserver': False }); def _before_vcard_create(self, kind, triplets, vcard=None): route_id = self.data.get('route_id', [vcard and vcard.route or None])[0] if route_id: if route_id not in self.session.config.routes: raise ValueError('Not a valid route ID: %s' % route_id) elif self.data.get('route-protocol', ['none'])[0] != 'none': route_id = self.session.config.routes.append({}) return { 'save_config': True, 'route_id': route_id } def _update_vcard_from_post(self, vcard, state=None): if not state: # When editing, this doesn't run first, so we invoke it now. state = self._before_vcard_create(vcard.kind, [], vcard=vcard) vcard.signature = self.data.get('signature', [''])[0] vcard.email = self.data.get('email', [None])[0] or vcard.email vcard.fn = self.data.get('name', [None])[0] or vcard.fn if not vcard.tag: with self.session.config._lock: tags = self.session.config.tags vcard.tag = tags.append({ 'name': vcard.email, 'slug': '%8.8x' % time.time(), 'type': 'profile', 'icon': 'icon-user', 'flag_msg_only': True, 'label': False, 'display': 'invisible' }) from mailpile.plugins.tags import Slugify tags[vcard.tag].slug = Slugify( 'account-%s' % vcard.email, tags=self.session.config.tags) route_id = state.get('route_id', None) if route_id is not None: self._configure_sending_route(vcard, route_id) self._configure_mail_sources(vcard) self._configure_security(vcard) def _after_vcard_create(self, kind, vcard, state): self._update_vcard_from_post(vcard, state=state) class EditProfile(AddProfile): """Edit a profile""" SYNOPSIS = (None, None, 'profiles/edit', None) HTTP_QUERY_VARS = dict_merge(AddProfile.HTTP_QUERY_VARS, { 'rid': 'update by x-mailpile-rid'}) def _vcard_to_post_vars(self, vcard): cp = vcard.crypto_policy or '' cf = vcard.crypto_format or '' vc_sig = vcard.signature default_sig = self._default_signature() pvars = { 'rid': vcard.random_uid, 'name': vcard.fn, 'email': vcard.email, 'signature': default_sig if (vc_sig is None) else vc_sig, 'password': '', 'route-protocol': 'none', 'route-auth_type': 'password', 'source-NEW-protocol': 'none', 'source-NEW-auth_type': 'password', 'security-pgp-key': vcard.pgp_key or '', 'security-best-effort-crypto': ('best-effort' in cp), 'security-autocrypt-crypto': ('autocrypt' in cp), 'security-use-autocrypt': ('autocrypt' in cf or 'autocrypt' in cp), 'security-always-sign': ('sign' in cp), 'security-always-encrypt': ('encrypt' in cp), 'security-attach-keys': ('send_keys' in cf), 'security-prefer-inline': ('prefer_inline' in cf), 'security-prefer-pgpmime': ('pgpmime' in cf), 'security-obscure-metadata': ('obscure_meta' in cf), 'security-openpgp-header-encrypt': ('openpgp_header:E' in cf or 'openpgp_header:SE' in cf), 'security-openpgp-header-sign': ('openpgp_header:S' in cf or 'openpgp_header:ES' in cf), 'security-openpgp-header-none': ('openpgp_header:N' in cf), 'security-publish-to-keyserver': ('publish' in cf) } route = self.session.config.routes.get(vcard.route or 'ha ha ha') if route: pvars.update({ 'route-protocol': route.protocol, 'route-host': route.host, 'route-port': route.port, 'route-username': route.username, 'route-password': route.password, 'route-auth_type': route.auth_type, 'route-command': route.command }) pvars['sources'] = vcard.sources() for sid in pvars['sources']: prefix = 'source-%s-' % sid source = self.session.config.sources.get(sid) disco = source.discovery info = {} for rvar in ('protocol', 'auth_type', 'host', 'port', 'username', 'password'): info[prefix + rvar] = source[rvar] dp = disco.policy if not info[prefix + 'auth_type']: info[prefix + 'auth_type'] = 'password' info[prefix + 'leave-on-server'] = (dp not in 'move') info[prefix + 'index-all-mail'] = (dp in ('move', 'sync', 'read') and disco.local_copy) info[prefix + 'enabled'] = source.enabled if source.protocol == 'imap_tls': info[prefix + 'protocol'] = 'imap' info[prefix + 'force-starttls'] = True else: info[prefix + 'force-starttls'] = False pvars.update(info) return pvars def command(self): idx = self._idx() # Make sure VCards are all loaded session, config = self.session, self.session.config # OK, fetch the VCard. safe_assert('rid' in self.data and len(self.data['rid']) == 1) vcard = config.vcards.get_vcard(self.data['rid'][0]) safe_assert(vcard) if self.data.get('_method') == 'POST': self._update_vcard_from_post(vcard) self._background_save(config=True) vcard.save() return self._success(_('Account Updated!'), self._vcard_to_post_vars(vcard)) else: return self._success(_('Edit Account'), dict_merge( self._form_defaults(), self._vcard_to_post_vars(vcard))) class RemoveProfile(ProfileVCard(RemoveVCard)): """Remove a profile""" HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = dict_merge(RemoveVCard.HTTP_QUERY_VARS, { 'rid': 'x-mailpile-rid of profile to remove'}) HTTP_POST_VARS = { 'rid': 'x-mailpile-rid of profile to remove', 'trash-email': 'If yes, move linked e-mail to Trash', 'delete-keys': 'If yes, delete linked PGP keys', 'delete-tags': 'If yes, remove linked Tags'} def _trash_email(self, vcard): trashes = self.session.config.get_tags(type='trash', default=[]) if vcard.tag and trashes: idx = self.session.config.index idx.add_tag(self.session, trashes[0]._key, msg_idxs=idx.TAGS.get(vcard.tag, set()), allow_message_id_clearing=True, conversation=False) def _delete_keys(self, vcard): if vcard.pgp_key: found = 0 for vc in self.session.config.vcards.find_vcards([], kinds=['profile']): if vc.pgp_key == vcard.pgp_key: found += 1 if found == 1: self._gnupg().delete_key(vcard.pgp_key) def _cleanup_tags(self, vcard, delete_tags=False): if vcard.tag: if delete_tags: from mailpile.plugins.tags import DeleteTag DeleteTag(self.session, arg=[vcard.tag]).run() else: self.session.config.tags[vcard.tag].type = 'attribute' self.session.config.tags[vcard.tag].display = 'invisible' self.session.config.tags[vcard.tag].label = True def _unique_usernames(self, vcard): config, usernames = self.session.config, set() if (vcard.route not in ('', None)) and config.routes[vcard.route].username: usernames.add(config.routes[vcard.route].username) for source_id in vcard.sources(): if config.sources[source_id].username: usernames.add(config.sources[source_id].username) for msid, source in config.sources.iteritems(): if (source.username in usernames) and (source.profile != vcard.random_uid): usernames.remove(source.username) for mrid, route in config.routes.iteritems(): if (route.username in usernames) and (mrid != vcard.route): usernames.remove(source.username) return usernames def _delete_credentials(self, vcard): # Check every stored credential; if it is in use by any route or source that # doesn't belong to this VCard, leave intact. Otherwise, delete. usernames = self._unique_usernames(vcard) oauth_tks = self.session.config.oauth.tokens for username in (set(oauth_tks.keys()) & usernames): del oauth_tks[username] secrets = self.session.config.secrets for username in (set(secrets.keys()) & usernames): del secrets[username] def _delete_routes(self, vcard): if vcard.route not in (None, ''): found = 0 for vc in self.session.config.vcards.find_vcards([], kinds=['profile']): if vc.route == vcard.route: found += 1 if found == 1: self.session.config.routes[vcard.route] = {} def _delete_sources(self, vcard, delete_tags=False): config, sources = self.session.config, self.session.config.sources for source_id in vcard.sources(): src = sources[source_id] tids = set() # Tell the worker to shut down, if any if source_id in config.mail_sources: config.mail_sources[source_id].quit() config.mail_sources[source_id].event.flags = Event.COMPLETE # Keep the reference to our local mailboxes in sys.mailbox for mbx_id, mbx_info in src.mailbox.iteritems(): if mbx_info.primary_tag: tids.add(mbx_info.primary_tag) if mbx_info.local and (mbx_info.path[:1] == '@'): config.sys.mailbox[mbx_info.path[1:]] = mbx_info.local # Reconfigure all the tags from mailpile.plugins.tags import DeleteTag if src.discovery.parent_tag: tids.add(src.discovery.parent_tag) for tid in tids: if ((tid in config.tags) and config.tags[tid].type in ('mailbox', 'profile')): config.tags[tid].type = 'tag' if delete_tags: DeleteTag(self.session, arg=[tid]).run() # Nuke it! sources[source_id] = { 'name': 'Deleted source', 'enabled': False} def _trash_email_is_safe(self, vcard): if vcard: for src_id in vcard.sources(): if self.session.config.sources[src_id].protocol == 'local': return False return True return False def command(self, *args, **kwargs): session, config = self.session, self.session.config if 'rid' in self.data: vcard = config.vcards.get_vcard(self.data['rid'][0]) else: vcard = None if vcard and self.data.get('_method', 'not-http').upper() != 'GET': if self.data.get('trash-email', [''])[0].lower() == 'yes': self._trash_email(vcard) if self.data.get('delete-keys', [''])[0].lower() == 'yes': self._delete_keys(vcard) delete_tags = (self.data.get('delete-tags', [''])[0].lower() == 'yes') self._delete_credentials(vcard) self._delete_routes(vcard) self._delete_sources(vcard, delete_tags=delete_tags) self._cleanup_tags(vcard, delete_tags=delete_tags) self._background_save(config=True, index=True) return RemoveVCard.command(self, *args, **kwargs) return self._success(_("Remove account"), result=dict_merge( self._form_defaults(), { 'rid': vcard.random_uid if vcard else None, 'trash_email_is_safe': self._trash_email_is_safe(vcard), 'profile': (self._vcard_list([vcard])['profiles'][0] if vcard else None)})) class ListProfiles(ProfileVCard(ListVCards)): """Find profiles""" SYNOPSIS = (None, 'profiles', 'profiles', '[--lines] []') class ProfileSet(ProfileVCard(VCardSet)): """Set contact lines, ensuring contact exists""" class ChooseFromAddress(Command): """Display a single vcard""" SYNOPSIS = (None, 'profiles/choose_from', 'profiles/choose_from', '') ORDER = ('Internals', 6) HTTP_CALLABLE = ('GET',) HTTP_QUERY_VARS = { 'mid': 'Message ID', 'email': 'E-mail address', 'no_from': 'Ignore From: lines' } def command(self): idx, vcards = self._idx(), self.session.config.vcards emails = [e for e in self.args if '@' in e] emails.extend(self.data.get('email', [])) messages = self._choose_messages( [m for m in self.args if '@' not in m] + ['=%s' % mid for mid in self.data.get('mid', [])] ) for msg_idx_pos in messages: try: msg_info = idx.get_msg_at_idx_pos(msg_idx_pos) msg_emails = (idx.expand_to_list(msg_info, field=idx.MSG_TO) + idx.expand_to_list(msg_info, field=idx.MSG_CC)) emails.extend(msg_emails) if 'no_from' not in self.data: emails.append(msg_info[idx.MSG_FROM]) except ValueError: pass addrs = [ai for ee in emails for ai in AddressHeaderParser(unicode_data=ee)] return self._success(_('Choosing from address'), result={ 'emails': addrs, 'from': vcards.choose_from_address(self.session.config, addrs) }) class ContentTxf(EmailTransform): def TransformOutgoing(self, sender, rcpts, msg, **kwargs): txf_matched, txf_continue = False, True profile = self._get_sender_profile(sender, kwargs) sig = profile.get('signature') if sig: part = self._get_first_part(msg, 'text/plain') if part is not None: msg_text = (part.get_payload(decode=True) or '\n\n' ).replace('\r', '').decode('utf-8') if '\n-- \n' not in msg_text: msg_text = msg_text.strip() + '\n\n-- \n' + sig try: msg_text.encode('us-ascii') need_utf8 = False except (UnicodeEncodeError, UnicodeDecodeError): msg_text = msg_text.encode('utf-8') need_utf8 = True part.set_payload(msg_text) if need_utf8: part.set_charset('utf-8') while 'content-transfer-encoding' in part: del part['content-transfer-encoding'] encoders.encode_base64(part) txf_matched = True return sender, rcpts, msg, txf_matched, txf_continue _plugins.register_commands(VCard, AddVCard, RemoveVCard, ListVCards, VCardAddLines, VCardSet, VCardRemoveLines) _plugins.register_commands(Contact, AddContact, RemoveContact, ListContacts, ContactSet, AddressSearch) _plugins.register_commands(Profile, AddProfile, EditProfile, RemoveProfile, ListProfiles, ProfileSet, ChooseFromAddress) _plugins.register_commands(ContactImport, ContactImporters) _plugins.register_outgoing_email_content_transform('100_sender_vc', ContentTxf) ================================================ FILE: mailpile/plugins/core.py ================================================ # These are the Mailpile core commands, the public "API" we expose for # searching, tagging and editing e-mail. # # FIXME: This should probably be broken into smaller modules # import datetime import json import os import random import re import socket import subprocess import sys import traceback import thread import threading import time import webbrowser import mailpile.util import mailpile.postinglist import mailpile.security as security import mailpile.platforms from mailpile.commands import * from mailpile.config.validators import WebRootCheck from mailpile.crypto.gpgi import GnuPG from mailpile.eventlog import Event from mailpile.i18n import gettext as _ from mailpile.i18n import ngettext as _n from mailpile.mailboxes import IsMailbox from mailpile.mailutils.emails import ClearParseCache, Email from mailpile.postinglist import GlobalPostingList from mailpile.plugins import PluginManager from mailpile.safe_popen import MakePopenUnsafe, MakePopenSafe from mailpile.search import MailIndex from mailpile.util import * from mailpile.vcard import AddressInfo from mailpile.vfs import vfs, FilePath _plugins = PluginManager(builtin=__file__) class Load(Command): """Load or reload the metadata index""" SYNOPSIS = (None, 'load', None, None) ORDER = ('Internals', 1) CONFIG_REQUIRED = False IS_INTERACTIVE = True def command(self, reset=True, wait=True, wait_all=False, quiet=False): try: if self._idx(reset=reset, wait=wait, wait_all=wait_all, quiet=quiet): return self._success(_('Loaded metadata index')) else: return self._error(_('Failed to load metadata index')) except IOError: return self._error(_('Failed to decrypt configuration, ' 'please log in!')) class Rescan(Command): """Add new messages to index""" SYNOPSIS = (None, 'rescan', 'rescan', '[full|vcards|vcards:|sources|mailboxes|both|mailbox:|]') ORDER = ('Internals', 2) LOG_PROGRESS = True HTTP_CALLABLE = ('POST',) HTTP_POST_VARS = { 'which': '[full|vcards|vcards:|both|mailboxes|sources|]' } def _progress(self, progress): self.session.ui.mark(progress) self.event.data["progress"].append((int(time.time()), progress)) def command(self, slowly=False, cron=False): session, config, idx = self.session, self.session.config, self._idx() self.event.data["progress"] = [] args = list(self.args) if 'which' in self.data: args.extend(self.data['which']) # Abort if we are out of disk space full_path = config.need_more_disk_space() if full_path: return self._error(_('Insufficient free space in %s') % full_path) # Pretend we're idle, to make rescan go fast fast. if not slowly: mailpile.util.LAST_USER_ACTIVITY = 0 # Cron always runs the rescan command, no matter what else if cron: self._run_rescan_command(session) a0lower = (args and args[0] or '').lower() if a0lower.startswith('vcards'): return self._success(_('Rescanned vCards'), result=self._rescan_vcards(session, args[0])) elif (a0lower in ('both', 'mailboxes', 'sources', 'editable') or a0lower.startswith('mailbox:')): return self._success(_('Rescanned mailboxes'), result=self._rescan_mailboxes(session, which=a0lower)) elif a0lower == 'full': config.flush_mbox_cache(session, wait=True) args.pop(0) # Clear the cache first, in case the user is flailing about ClearParseCache(full=True) msg_idxs = self._choose_messages(args) if msg_idxs: for msg_idx_pos in msg_idxs: e = Email(idx, msg_idx_pos) try: self._progress('Re-indexing %s' % e.msg_mid()) idx.index_email(self.session, e) except KeyboardInterrupt: self._progress('Interrupted') raise except: self._ignore_exception() session.ui.warning(_('Failed to reindex: %s' ) % e.msg_mid()) self.event.data["messages"] = len(msg_idxs) self.session.config.event_log.log_event(self.event) self._background_save(index=True) return self._success(_('Indexed %d messages') % len(msg_idxs), result={'messages': len(msg_idxs)}) else: deadline = (int(time.time() + 0.75 * config.prefs.rescan_interval) if cron else None) if 'rescan' in config._running: return self._success(_('Rescan already in progress')) config._running['rescan'] = True try: results = {} results.update(self._rescan_vcards(session, 'vcards')) results.update(self._rescan_mailboxes(session, deadline=deadline, which=('mailboxes' if cron else 'both'), force=(not cron and not slowly))) self.event.data.update(results) self.session.config.event_log.log_event(self.event) if 'aborted' in results: self._progress('Aborted') raise KeyboardInterrupt() return self._success(_('Rescanned vCards and mailboxes'), result=results) except KeyboardInterrupt: self._progress('Interrupted') return self._error(_('User aborted'), info=results) finally: self._progress("Rescan complete") del config._running['rescan'] def _rescan_vcards(self, session, which): from mailpile.plugins import PluginManager config = session.config imported = 0 importer_cfgs = config.prefs.vcard.importers which_spec = which.split(':') importers = [] try: self._progress(_('Rescanning: %s') % 'vcards') for importer in PluginManager.VCARD_IMPORTERS.values(): if (len(which_spec) > 1 and which_spec[1] != importer.SHORT_NAME): continue importers.append(importer.SHORT_NAME) for cfg in importer_cfgs.get(importer.SHORT_NAME, []): if cfg: imp = importer(session, cfg) self._progress(_('Importing VCards from: %s') % imp) imported += imp.import_vcards(session, config.vcards) if mailpile.util.QUITTING: return { 'vcards': imported, 'vcard_sources': importers, 'aborted': True} except KeyboardInterrupt: return { 'vcards': imported, 'vcard_sources': importers, 'aborted': True} return { 'vcards': imported, 'vcard_sources': importers} def _run_rescan_command(self, session, timeout=120): pre_command = session.config.prefs.rescan_command if pre_command and not mailpile.util.QUITTING: self._progress(_('Running: %s') % pre_command) if not ('|' in pre_command or '&' in pre_command or ';' in pre_command): pre_command = pre_command.split() cmd = subprocess.Popen(pre_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not isinstance(pre_command, list)) countdown = [timeout] def eat(fmt, fd): for line in fd: session.ui.notify(fmt % line.strip()) countdown[0] = timeout for t in [ threading.Thread(target=eat, args=['E: %s', cmd.stderr]), threading.Thread(target=eat, args=['O: %s', cmd.stdout]) ]: t.daemon = True t.start() try: while countdown[0] > 0: countdown[0] -= 1 if cmd.poll() is not None: rv = cmd.wait() if rv != 0: session.ui.notify(_('Rescan command returned %d') % rv) return elif mailpile.util.QUITTING: return time.sleep(1) finally: if cmd.poll() is None: session.ui.notify(_('Aborting rescan command')) cmd.terminate() time.sleep(0.2) if cmd.poll() is None: cmd.kill() # NOTE: For some reason we were using the un-safe Popen before, not sure # if that matters. Leaving this commented out for now for reference. # # try: # MakePopenUnsafe() # subprocess.check_call(pre_command, shell=True) # finally: # MakePopenSafe() def _rescan_mailboxes(self, session, which='mailboxes', force=True, deadline=None): import mailpile.mail_source config = session.config idx = self._idx() msg_count = 0 mbox_count = 0 rv = True try: self._progress(_('Rescanning: %s') % which) self._run_rescan_command(session) if which.startswith('mailbox:'): only = which.split(':')[1] which = 'mailboxes' else: only = None msg_count = 1 if which in ('both', 'mailboxes', 'editable'): if only or which == 'editable': mailboxes = config.get_mailboxes() else: # This combination of arguments will ignore mailboxes linked to # active mail sources, but include the local caches of sources # that have been disabled. mailboxes = config.get_mailboxes(with_mail_source=False, mail_source_locals=True) for fid, fpath, sc in mailboxes: if mailpile.util.QUITTING: break if fpath == '/dev/null': continue if only and (only != fpath) and (only != fid): continue try: self._progress(_('Rescanning: %s %s') % (fid, fpath)) rescan_args = { 'event': self.event, 'deadline': deadline, 'force': force} if which == 'editable': count = idx.scan_mailbox(session, fid, fpath, config.open_mailbox, process_new=False, editable=True, **rescan_args) else: count = idx.scan_mailbox(session, fid, fpath, config.open_mailbox, **rescan_args) except ValueError: self._ignore_exception() count = -1 if count < 0: session.ui.warning(_('Failed to rescan: %s') % FilePath(fpath).display()) elif count > 0: msg_count += count mbox_count += 1 session.ui.mark('\n') if which in ('both', 'sources'): ocount = msg_count - 1 while ocount != msg_count: ocount = msg_count src_ids = config.sources.keys() src_ids.sort(key=lambda k: random.randint(0, 100)) for src_id in src_ids: try: src = config.get_mail_source(src_id, start=True) if mailpile.util.QUITTING: ocount = msg_count break self._progress(_('Rescanning: %s') % (src, )) (messages, mailboxes) = src.rescan_now(session) except ValueError: messages = mailboxes = 0 if messages > 0: msg_count += messages mbox_count += mailboxes session.ui.mark('\n') if not session.ui.interactive: break except (KeyboardInterrupt, subprocess.CalledProcessError) as e: return { 'aborted': True, 'messages': msg_count, 'mailboxes': mbox_count} finally: if msg_count: session.ui.mark('\n') if msg_count < 500: self._background_save(index=True) else: self._background_save(index_full=True) else: self._progress(_('Nothing changed')) return { 'messages': msg_count, 'mailboxes': mbox_count} class Optimize(Command): """Optimize the keyword search index""" SYNOPSIS = (None, 'optimize', None, '[harder]') ORDER = ('Internals', 3) def command(self, slowly=False): try: if not slowly: mailpile.util.LAST_USER_ACTIVITY = 0 self._idx().save(self.session) GlobalPostingList.Optimize(self.session, self._idx(), force=('harder' in self.args)) return self._success(_('Optimized search engine')) except KeyboardInterrupt: return self._error(_('Aborted')) class DeleteMessages(Command): """Delete one or more messages.""" SYNOPSIS = (None, 'delete', 'message/delete', '[--keep] ') ORDER = ('Searching', 99) IS_USER_ACTIVITY = True def command(self, slowly=False): idx = self._idx() args = list(self.args) keep = 0 while '--keep' in args: args.remove('--keep') keep += 1 # We group messages by mailbox and delete in batches. This should # avoid loading all the mailboxes into RAM at once, which is a big # deal on larger setups. targets = [Email(idx, mi) for mi in self._choose_messages(args)] msg_ptr_pairs = [ (e, e.index.unique_mbox_ids(e.get_msg_info())) for e in targets] if 'deletion' in self.session.config.sys.debug: self.session.ui.debug('Targets: %s' % msg_ptr_pairs) # Message are sorted so the ones present in the most mailboxes # are listed first. msg_ptr_pairs.sort(key=lambda mpp: (-len(mpp[1]), mpp[0])) deleted, failed = [], [] while msg_ptr_pairs: # Pick the largest set of mailboxes we have yet to delete from mid_set = msg_ptr_pairs[0][1] # Pick all the messages contained in this set of mailboxes messages = [e for e, mids in msg_ptr_pairs if ((mid_set | mids) == mid_set)] # Go delete them! mailboxes = [] for e in messages: msg_idx = e.msg_idx_pos del_ok, mboxes = e.delete_message(self.session, flush=False, keep=keep) mailboxes.extend(mboxes) if del_ok: deleted.append(msg_idx) else: failed.append(msg_idx) # This will actually delete from mboxes, etc. for m in set(mailboxes): with m: m.flush() # OK, these are done, reduce our target list msg_ptr_pairs = [(e, mids) for e, mids in msg_ptr_pairs if ((mid_set | mids) != mid_set)] # FIXME: Trigger a background rescan of affected mailboxes, as # the flush() above may have broken our pointers. result = {'deleted': deleted} if failed: result['failed'] = failed return self._error(_('Could not delete all messages'), result=result) return self._success(_('Deleted %d messages') % len(deleted), result=result) class BrowseOrLaunch(Command): """Launch browser and exit, if already running""" SYNOPSIS = (None, 'browse_or_launch', None, None) ORDER = ('Internals', 5) CONFIG_REQUIRED = False RAISES = (KeyboardInterrupt,) @classmethod def Browse(cls, sspec): http_url = ('http://%s:%s%s/' % sspec ).replace('//0.0.0.0:', '//localhost:') try: MakePopenUnsafe() webbrowser.open(http_url) return http_url except: pass finally: MakePopenSafe() return False def command(self): config = self.session.config if config.http_worker: sspec = config.http_worker.sspec else: sspec = (config.sys.http_host, config.sys.http_port, config.sys.http_path or '') try: socket.create_connection(sspec[:2]) self.Browse(sspec) os._exit(127) except IOError: pass return self._success(_('Launching Mailpile'), result=True) class RunWWW(Command): """Just run the web server""" SYNOPSIS = (None, 'www', None, '[]') ORDER = ('Internals', 5) CONFIG_REQUIRED = False def command(self): config = self.session.config ospec = (config.sys.http_host, config.sys.http_port, config.sys.http_path) if self.args: host, portpath = self.args[0].split('://')[-1].split(':', 1) port, path = (portpath+'/').split('/', 1) port = int(port) sspec = (host, port, WebRootCheck(path)) else: sspec = ospec if self.session.config.http_worker: self.session.config.http_worker.quit(join=True) self.session.config.http_worker = None self.session.config.prepare_workers(self.session, httpd_spec=tuple(sspec), daemons=True) if config.http_worker: sspec = config.http_worker.httpd.sspec http_url = 'http://%s:%s%s/' % sspec if sspec != ospec: (config.sys.http_host, config.sys.http_port, config.sys.http_path) = sspec self._background_save(config=True) return self._success(_('Moved the web server to %s' ) % http_url) else: return self._success(_('Started the web server on %s' ) % http_url) else: return self._error(_('Failed to start the web server')) class Cleanup(Command): """Perform cleanup actions (runs before shutdown)""" SYNOPSIS = (None, 'cleanup', None, "") ORDER = ('Internals', 5) CONFIG_REQUIRED = False SPLIT_ARG = False TASKS = [] @classmethod def AddTask(cls, task, last=False, first=False): safe_assert(not (first and last)) if (first or last) and not cls.TASKS: cls.TASKS = [lambda: True] if first: cls.TASKS.insert(0, task) elif last: cls.TASKS.append(task) else: cls.TASKS.insert(len(cls.TASKS) - 1, task) def command(self): while self.TASKS: try: self.TASKS.pop(0)() except: traceback.print_exc() pass return self._success(_('Performed shutdown tasks')) class WritePID(Command): """Write the PID to a file""" SYNOPSIS = (None, 'pidfile', None, "") ORDER = ('Internals', 5) CONFIG_REQUIRED = False SPLIT_ARG = False def command(self): filename = self.args[0] with vfs.open(filename, 'w') as fd: fd.write('%d' % os.getpid()) Cleanup.AddTask(lambda: os.unlink(filename), last=True) return self._success(_('Wrote PID to %s') % self.args) class RenderPage(Command): """Does nothing, for use by semi-static jinja2 pages""" SYNOPSIS = (None, None, 'page', None) ORDER = ('Internals', 6) CONFIG_REQUIRED = False SPLIT_ARG = False HTTP_STRICT_VARS = False IS_USER_ACTIVITY = True def template_path(self, ttype, template_id=None, **kwargs): if not template_id: template_id = '%s/%s' % (self.SYNOPSIS[2], self.args and self.args[0] or '') return Command.template_path(self, ttype, template_id=template_id, **kwargs) def command(self): return self._success(_('Rendered the page'), result={ 'path': (self.args and self.args[0] or ''), 'data': self.data }) class ProgramStatus(Command): """Display list of running threads, locks and outstanding events.""" SYNOPSIS = (None, 'ps', 'ps', None) ORDER = ('Internals', 5) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False LOG_NOTHING = True class CommandResult(Command.CommandResult): def as_text(self): now = time.time() sessions = self.result.get('sessions') if sessions: sessions = '\n'.join(sorted([' %s/%s = %s (%ds)' % (us['sessionid'], us['userdata'], us['userinfo'], now - us['timestamp']) for us in sessions])) else: sessions = ' ' + _('Nothing Found') ievents = self.result.get('ievents') cevents = self.result.get('cevents') if cevents: cevents = '\n'.join([' %s' % (e.as_text(compact=True),) for e in cevents]) else: cevents = ' ' + _('Nothing Found') ievents = self.result.get('ievents') if ievents: ievents = '\n'.join([' %s' % (e.as_text(compact=True),) for e in ievents]) else: ievents = ' ' + _('Nothing Found') threads = self.result.get('threads') if threads: threads = '\n'.join(sorted([(' ' + str(t)) for t in threads])) else: threads = _('Nothing Found') locks = self.result.get('locks') if locks: locks = '\n'.join(sorted([(' %s.%s is %slocked' ) % (l[0], l[1], '' if l[2] else 'un') for l in locks])) else: locks = _('Nothing Found') return ('Recent events:\n%s\n\n' 'Events in progress:\n%s\n\n' 'Live sessions:\n%s\n\n' 'Postinglist timers:\n%s\n\n' 'Threads: (bg delay %.3fs, live=%s, httpd=%s)\n%s\n\n' 'Locks:\n%s' ) % (cevents, ievents, sessions, self.result['pl_timers'], self.result['delay'], self.result['live'], self.result['httpd'], threads, locks) def command(self, args=None): import mailpile.auth import mailpile.mail_source import mailpile.plugins.compose import mailpile.plugins.contacts config = self.session.config try: idx = config.index locks = [ ('config.index', '_lock', idx._lock._is_owned()), ('config.index', '_save_lock', idx._save_lock._is_owned()) ] except AttributeError: locks = [] if config.vcards: locks.extend([ ('config.vcards', '_lock', config.vcards._lock._is_owned()), ]) locks.extend([ ('config', '_lock', config._lock._is_owned()), ('mailpile.plugins.compose', 'GLOBAL_EDITING_LOCK', mailpile.plugins.compose.GLOBAL_EDITING_LOCK._is_owned()), ('mailpile.plugins.contacts', 'GLOBAL_VCARD_LOCK', mailpile.plugins.contacts.GLOBAL_VCARD_LOCK._is_owned()), ('mailpile.postinglist', 'PLC_CACHE_LOCK', mailpile.postinglist.PLC_CACHE_LOCK.locked()), ('mailpile.postinglist', 'GLOBAL_POSTING_LOCK', mailpile.postinglist.GLOBAL_POSTING_LOCK._is_owned()), ('mailpile.postinglist', 'GLOBAL_OPTIMIZE_LOCK', mailpile.postinglist.GLOBAL_OPTIMIZE_LOCK.locked()), ('mailpile.postinglist', 'GLOBAL_GPL_LOCK', mailpile.postinglist.GLOBAL_GPL_LOCK._is_owned())]) threads = threading.enumerate() for thread in threads: try: if hasattr(thread, 'lock'): locks.append([thread, 'lock', thread.lock]) if hasattr(thread, '_lock'): locks.append([thread, '_lock', thread._lock]) if locks and hasattr(locks[-1][-1], 'locked'): locks[-1][-1] = locks[-1][-1].locked() elif locks and hasattr(locks[-1][-1], '_is_owned'): locks[-1][-1] = locks[-1][-1]._is_owned() except AttributeError: pass import mailpile.auth import mailpile.httpd result = { 'sessions': [{'sessionid': k, 'timestamp': v.ts, 'userdata': v.data, 'userinfo': v.auth} for k, v in mailpile.auth.SESSION_CACHE.iteritems()], 'pl_timers': mailpile.postinglist.TIMERS, 'delay': play_nice_with_threads(sleep=False), 'live': mailpile.util.LIVE_USER_ACTIVITIES, 'httpd': mailpile.httpd.LIVE_HTTP_REQUESTS, 'threads': threads, 'locks': sorted(locks) } if config.event_log: result.update({ 'cevents': list(config.event_log.events(flag='c'))[-10:], 'ievents': config.event_log.incomplete(), }) return self._success(_("Listed events, threads, and locks"), result=result) class CronStatus(Command): """Manually edit or display the background job schedule""" SYNOPSIS = (None, 'cron', None, "[ <--trigger|--interval |--postpone >]") ORDER = ('Internals', 4) IS_USER_ACTIVITY = False class CommandResult(Command.CommandResult): def as_text(self): def _t(dt): return '%4.4d-%2.2d-%2.2d %2.2d:%2.2d' % ( dt.year, dt.month, dt.day, dt.hour, dt.minute) fmt = ' %-23s %8s %-16s %-16s %s' lines = [ 'Background CRON last ran at %s.' % _t( datetime.datetime.fromtimestamp(self.result['last_run'])), 'Current schedule:', '', fmt % ('JOB', 'INTERVAL', 'LAST RUN', 'NEXT RUN', 'STATUS')] for job_name, interval, func, last, status in self.result['jobs']: lines.append(fmt % ( job_name, interval, (_t(datetime.datetime.fromtimestamp(last)) if (status != 'new') else ''), _t(datetime.datetime.fromtimestamp(last + interval)), status)) return '\n'.join(lines) def command(self, args=None): config = self.session.config args = args if (args is not None) else list(self.args) now = int(time.time()) if args: job = args.pop(0) while args: op = args.pop(0).lower().replace('-', '') if op == 'interval': interval = int(args.pop(0)) config.cron_worker.schedule[job][1] = interval elif op == 'trigger': interval = config.cron_worker.schedule[job][1] config.cron_worker.schedule[job][3] = now - interval elif op == 'postpone': hours = float(args.pop(0)) config.cron_worker.schedule[job][3] += int(hours * 3600) else: raise NotImplementedError('Unknown op: %s' % op) return self._success( _("Displayed CRON schedule"), result={ 'last_run': config.cron_worker.last_run, 'jobs': config.cron_worker.schedule.values()}) class HealthCheck(Command): """Check and report app health""" SYNOPSIS = (None, 'health', None, "") ORDER = ('Internals', 4) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False # We cache our health event, so it can be updated by the class methods. health_event = None def _create_event(self): if HealthCheck.health_event is not None: self.event = HealthCheck.health_event else: Command._create_event(self) self.event.data['starttime'] = int(time.time()) self.event.data['problems'] = {} self.event.data['healthy'] = True HealthCheck.health_event = self.event # Cancel any obsolete HealthCheck events we find if self.session.config.event_log: for ev in self.session.config.event_log.events(): if (ev.source == self.event.source and ev.event_id != self.event.event_id): ev.flags = ev.COMPLETE self.session.config.event_log.log_event(ev) @classmethod def _mem_check(cls, session, config): if config.detected_memory_corruption: return _('Memory corruption detected') + '!' return False @classmethod def _disk_check(cls, session, config): if config.need_more_disk_space(): return _('Insufficient free disk space') + '.' return False @classmethod def _readonly_check(cls, session, config): from mailpile.security import _lockdown_basic if _lockdown_basic(config): return _('Your Mailpile is read-only!') return False @classmethod def check(cls, session, config): # Check all the things! The order here matters, more critical things # should be reported last as they will determine the final message. if not cls.health_event: return False messages = [] problems = cls.health_event.data['problems'] was_healthy = cls.health_event.data['healthy'] old_problems = ' '.join(sorted(problems.keys())) now_healthy = True for crit, name, check in ((True, 'disk', cls._disk_check), (True, 'memcheck', cls._mem_check), (True, 'readonly', cls._readonly_check)): message = check(session, config) if message: problems[name] = message messages.append(message) if crit: now_healthy = False elif name in problems: del problems[name] cls.health_event.data['healthy'] = now_healthy if messages: cls.health_event.message = ' '.join(messages[-2:]) cls.health_event.flags = cls.health_event.RUNNING else: cls.health_event.message = _('We are healthy!') cls.health_event.flags = cls.health_event.COMPLETE # Only record changes to the event log new_problems = ' '.join(sorted(problems.keys())) if old_problems != new_problems and config.event_log: config.event_log.log_event(cls.health_event) return True def command(self, args=None): self.check(self.session, self.session.config) return self._success(self.event.message, result=self.event) class GpgCommand(Command): """Interact with GPG directly""" SYNOPSIS = (None, 'gpg', None, "") ORDER = ('Internals', 4) IS_USER_ACTIVITY = True class CommandResult(Command.CommandResult): def as_text(self): if self.result: return '%s\n\n%s' % ( (self.result['stdout'] or _('(no output)')).strip(), self.message) return '%s' % self.message def command(self, args=None): args = list((args is None) and self.args or args or []) gnupg = self._gnupg() gnupg_args = gnupg.common_args(interactive=True) binary = gnupg.gpgbinary rv = '(unknown)' def _shellquote(s): return "'" + s.replace("'", "'\\''") + "'" with self.session.ui.term: try: self.session.ui.block() if (self.session.ui.interactive and self.session.ui.render_mode == 'text'): rv = os.system(' '.join(_shellquote(a) for a in (gnupg_args + args))) stdout = None else: sp = subprocess.Popen( gnupg_args + ['--batch', '--no-tty'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (stdout, stderr) = sp.communicate(input='') rv = sp.wait() from mailpile.plugins.vcard_gnupg import PGPKeysImportAsVCards PGPKeysImportAsVCards(self.session).run() except: self.session.ui.unblock() return self._success(_("That was fun!") + ' ' + _("%s returned: %s") % (binary, rv), result={'binary': binary, 'stdout': stdout, 'returned': rv}) class ListDir(Command): """Display working directory listing""" SYNOPSIS = (None, 'ls', 'browse', "[-a] [-d] [ ...]") ORDER = ('Internals', 5) CONFIG_REQUIRED = False IS_USER_ACTIVITY = True COMMAND_SECURITY = security.CC_BROWSE_FILESYSTEM class CommandResult(Command.CommandResult): def as_text(self): if self.result and self.result['entries']: lines = [] for i in self.result['entries']: sz = i.get('bytes') dn = i['display_name'] dp = '' if (i['display_path'].endswith(dn) ) else i['display_path'] dn += '/' if i.get('flag_directory') else '' lines.append(('%12.12s %s%-20s %s' ) % ('' if (sz is None) else sz, '>' if i.get('flag_mailsource') else '*' if i.get('flag_mailbox') else ' ', dn, dp)) return '\n'.join(lines) else: return _('Nothing Found') def command(self, args=None): args = list((args is None) and self.args or args or []) flags = [f for f in args if f[:1] == '-'] args = [a for a in args if a[:1] != '-'] if '_method' in self.data: args = ['/' + '/'.join(args)] if not args: args = ['.'] def lsf(f): info = {'path': f} try: info = vfs.getinfo(f, self.session.config) info['icon'] = '' for k in info.get('flags', []): info['flag_%s' % unicode(k).lower().replace('.', '_') ] = True except (OSError, IOError, UnicodeDecodeError): info['flag_error'] = True return info def ls(p): return [lsf(vfs.path_join(p, f)) for f in vfs.listdir(p) if '-a' in flags or f.raw_fp[:1] != '.'] file_list = [] errors = 0 for path in args: if (security.forbid_command(self, security.CC_ACCESS_FILESYSTEM) and (path != '/') and (not path.endswith('$')) and ('$/' not in path)): continue try: path = os.path.expanduser(path.encode('utf-8')) if vfs.isdir(path) and '*' not in path: file_list.extend(ls(path)) else: for p in vfs.glob(path): if vfs.isdir(p) and '-d' not in flags: file_list.extend(ls(p)) else: file_list.append(lsf(p)) except (socket.error, socket.gaierror) as e: return self._error(_('Network error: %s') % e) except (OSError, IOError, UnicodeDecodeError) as e: errors += 1 if errors and not file_list: traceback.print_exc() return self._error(_('Failed to list: %s') % e) id_src_map = self.session.config.find_mboxids_and_sources_by_path( *[unicode(f['path']) for f in file_list]) for info in file_list: path = unicode(info['path']) mid_src = id_src_map.get(path) if mid_src: mid, src = mid_src if src: info['source'] = src._key if src and src.mailbox[mid] and src.mailbox[mid].primary_tag: tid = src.mailbox[mid].primary_tag if tid in self.session.config.tags: info['tag'] = self.session.config.tags[tid].slug info['icon'] = self.session.config.tags[tid].icon elif info.get('flag_mailsource'): if path.startswith('/src:'): info['source'] = path[5:] file_list.sort(key=lambda i: i['display_path'].lower()) return self._success(_('Listed %d files or directories' ) % len(file_list), result={ 'path': args[0] if (len(args) == 1) else args, 'name': vfs.display_name(args[0], self.session.config), 'entries': file_list }) class ChangeDir(ListDir): """Change working directory""" SYNOPSIS = (None, 'cd', None, "<.../new/path/...>") ORDER = ('Internals', 5) CONFIG_REQUIRED = False IS_USER_ACTIVITY = True COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM def command(self, args=None): try: args = list((args is None) and self.args or args or []) os.chdir(FilePath.unalias( os.path.expanduser(args.pop(0).encode('utf-8')))) return ListDir.command(self, args=['.']) except (OSError, IOError, UnicodeEncodeError) as e: return self._error(_('Failed to change directories: %s') % e) class CatFile(Command): """Dump the contents of a file, decrypting if necessary""" SYNOPSIS = (None, 'cat', None, " [>/path/to/output]") ORDER = ('Internals', 5) CONFIG_REQUIRED = False IS_USER_ACTIVITY = True COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM class CommandResult(Command.CommandResult): def as_text(self): if isinstance(self.result, list): return ''.join(self.result) else: return '' def command(self, args=None): lines = [] files = list(args or self.args) target = tfd = None if files and files[-1] and files[-1][:1] == '>': target = files.pop(-1)[1:] if vfs.exists(target): return self._error(_('That file already exists: %s' ) % target) tfd = vfs.open(target, 'wb') cb = lambda ll: [tfd.write(l) for l in ll] else: cb = lambda ll: lines.extend((l.decode('utf-8') for l in ll)) for fn in files: with vfs.open(fn, 'r') as fd: def errors(where): self.session.ui.error('Decrypt failed at %d' % where) decrypt_and_parse_lines(fd, cb, self.session.config, newlines=True, decode=None, gpgi=self._gnupg(), _raise=False, error_cb=errors) if tfd: tfd.close() return self._success(_('Dumped to %s: %s' ) % (target, ', '.join(files))) else: return self._success(_('Dumped: %s') % ', '.join(files), result=lines) ##[ Configuration commands ]################################################### class ListLanguages(Command): """List available languages""" SYNOPSIS = (None, 'languages', 'settings/languages', '') ORDER = ('Config', 1) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False HTTP_CALLABLE = ('GET', ) def command(self): from mailpile.i18n import ListTranslations langs = ListTranslations(self.session.config) return self._success(_('Listed available translations'), result=sorted([(l, langs[l]) for l in langs])) class ConfigSet(Command): """Change a setting""" SYNOPSIS = ('S', 'set', 'settings/set', '[--force] ') ORDER = ('Config', 1) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False SPLIT_ARG = False HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_STRICT_VARS = False HTTP_POST_VARS = { '_section': 'common section, create if needed', 'section.variable': 'value|json-string' } def command(self): from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD config = self.session.config args = list(self.args) arg = ' '.join(args) ops = [] on_cli = (self.data.get('_method', 'CLI') == 'CLI') force = False if arg.startswith('--force '): if not on_cli: raise ValueError('The --force flag only works on the CLI') force = True arg = arg[8:] if not force: fb = security.forbid_command(self, security.CC_CHANGE_CONFIG) if fb: return self._error(fb) if not config.loaded_config: self.session.ui.warning(_('WARNING: Any changes will ' 'be overwritten on login')) section = self.data.get('_section', [''])[0] if section: # Make sure section exists ops.append((section, '!CREATE_SECTION')) for var in self.data.keys(): if (var in ('_section', '_method', 'context', 'csrf') or var.startswith('ui_')): continue sep = '/' if ('/' in (section+var)) else '.' svar = (section+sep+var) if section else var parts = svar.split(sep) if parts[0] in config.rules: if svar.endswith('[]'): ops.append((svar[:-2], json.dumps(self.data[var]))) else: ops.append((svar, self.data[var][0])) else: raise ValueError(_('Invalid section or variable: %s') % var) if args: if '=' in arg: # Backwards compatiblity with the old 'var = value' syntax. var, value = [s.strip() for s in arg.split('=', 1)] var = var.replace(': ', '.').replace(':', '.').replace(' ', '') else: var, value = arg.split(' ', 1) ops.append((var, value)) # Access controls... if not force: for path, value in ops: fb = security.forbid_config_change(config, path) if fb: return self._error(fb) elif path == 'master_key' and config.get_master_key(): return self._error(_('I refuse to change the master key!')) # We don't have transactions really, but making sure the HTTPD # is idle (aside from this request) will definitely help. with BLOCK_HTTPD_LOCK, Idle_HTTPD(): updated = {} for path, value in ops: if not force: if path == 'master_key' and config.get_master_key(): raise ValueError('Need --force to change master key.') if path == 'sys.http_no_auth': raise ValueError('Need --force to change auth policy.') value = value.strip() if value == '{None}': value = None elif value == '{Blank}': value = '' elif value == '{False}': value = False elif value == '{True}': value = True elif value[:1] in ('{', '[') and value[-1:] in ( ']', '}'): value = json.loads(value) try: try: cfg, var = config.walk(path.strip(), parent=1) if value == '!CREATE_SECTION': if var not in cfg: cfg[var] = {} else: cfg[var] = value updated[path] = value except IndexError: cfg, v1, v2 = config.walk(path.strip(), parent=2) cfg[v1] = {v2: value} except TypeError: raise ValueError('Could not set variable: %s' % path) if config.loaded_config: self._background_save(config=True) return self._success(_('Updated your settings'), result=updated) class ConfigAdd(Command): """Add a new value to a list (or ordered dict) setting""" SYNOPSIS = (None, 'append', 'settings/add', ' ') ORDER = ('Config', 1) SPLIT_ARG = False HTTP_CALLABLE = ('POST', 'UPDATE') HTTP_STRICT_VARS = False HTTP_POST_VARS = { 'section.variable': 'value|json-string', } IS_USER_ACTIVITY = True COMMAND_SECURITY = security.CC_CHANGE_CONFIG def command(self): from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD config = self.session.config args = list(self.args) ops = [] for var in self.data.keys(): parts = ('.' in var) and var.split('.') or var.split('/') if parts[0] in config.rules: ops.append((var, self.data[var][0])) if args: arg = ' '.join(args) if '=' in arg: # Backwards compatible with the old 'var = value' syntax. var, value = [s.strip() for s in arg.split('=', 1)] var = var.replace(': ', '.').replace(':', '.').replace(' ', '') else: var, value = arg.split(' ', 1) ops.append((var, value)) # Access controls... for path, value in ops: fb = security.forbid_config_change(config, path) if fb: return self._error(fb) elif path == 'master_key' and config.get_master_key(): return self._error(_('I refuse to change the master key!')) # We don't have transactions really, but making sure the HTTPD # is idle (aside from this request) will definitely help. with BLOCK_HTTPD_LOCK, Idle_HTTPD(): updated = {} for path, value in ops: value = value.strip() if value.startswith('{') or value.startswith('['): value = json.loads(value) cfg, var = config.walk(path.strip(), parent=1) cfg[var].append(value) updated[path] = value if updated: self._background_save(config=True) return self._success(_('Updated your settings'), result=updated) class ConfigUnset(Command): """Reset one or more settings to their defaults""" SYNOPSIS = ('U', 'unset', 'settings/unset', '') ORDER = ('Config', 2) HTTP_CALLABLE = ('POST', ) HTTP_POST_VARS = { 'var': 'section.variables' } IS_USER_ACTIVITY = True COMMAND_SECURITY = security.CC_CHANGE_CONFIG def command(self): from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD session, config = self.session, self.session.config def unset(cfg, key): if isinstance(cfg[key], dict): if '_any' in cfg[key].rules: for skey in cfg[key].keys(): del cfg[key][skey] else: for skey in cfg[key].keys(): unset(cfg[key], skey) elif isinstance(cfg[key], list): cfg[key] = [] else: del cfg[key] # Access controls... vlist = list(self.args) + (self.data.get('var', None) or []) # Access controls... for v in vlist: fb = security.forbid_config_change(config, v) if fb: return self._error(fb) elif v == 'master_key' and config.get_master_key(): return self._error(_('I refuse to change the master key!')) # We don't have transactions really, but making sure the HTTPD # is idle (aside from this request) will definitely help. with BLOCK_HTTPD_LOCK, Idle_HTTPD(): updated = [] for v in vlist: cfg, vn = config.walk(v, parent=True) unset(cfg, vn) updated.append(v) if updated: self._background_save(config=True) return self._success(_('Reset to default values'), result=updated) class ConfigPrint(Command): """Print one or more settings""" SYNOPSIS = ('P', 'print', 'settings', '[-short|-secrets|-flat] ') ORDER = ('Config', 3) CONFIG_REQUIRED = False IS_USER_ACTIVITY = False HTTP_CALLABLE = ('GET', 'POST') HTTP_QUERY_VARS = { 'var': 'section.variable', 'short': 'Set True to omit unchanged values (defaults)', 'secrets': 'Set True to show passwords and other secrets' } HTTP_POST_VARS = { 'user': 'Authenticate as user', 'pass': 'Authenticate with password' } def _maybe_all(self, list_all, data, key_types, recurse, sanitize): if isinstance(data, (dict, list)) and list_all: rv = {} for key in data.all_keys(): if [t for t in data.key_types(key) if t not in key_types]: # Silently omit things that are considered sensitive continue rv[key] = data[key] if hasattr(rv[key], 'all_keys'): if recurse: rv[key] = self._maybe_all(True, rv[key], key_types, recurse, sanitize) else: if 'name' in rv[key]: rv[key] = '{ ..(%s).. }' % rv[key]['name'] elif 'description' in rv[key]: rv[key] = '{ ..(%s).. }' % rv[key]['description'] elif 'host' in rv[key]: rv[key] = '{ ..(%s).. }' % rv[key]['host'] else: rv[key] = '{ ... }' elif (sanitize and key.lower()[:4] in ('pass', 'secr', 'obfu')): rv[key] = '(SUPPRESSED)' return rv return data def command(self): session, config = self.session, self.session.config result = {} invalid = [] args = list(self.args) recurse = not self.data.get('flat', ['-flat' in args])[0] list_all = not self.data.get('short', ['-short' in args])[0] sanitize = not self.data.get('secrets', ['-secrets' in args])[0] if security.forbid_command(self, security.CC_LIST_PRIVATE_DATA): sanitize = True # FIXME: Shouldn't we suppress critical variables as well? key_types = ['public', 'critical'] access_denied = False if self.data.get('_method') == 'POST': if 'pass' in self.data: from mailpile.auth import CheckPassword password = self.data['pass'][0] auth_user = CheckPassword(config, self.data.get('user', [None])[0], password) if auth_user == 'DEFAULT': key_types += ['key'] result['_auth_user'] = auth_user result['_auth_pass'] = password for key in (args + self.data.get('var', [])): if key in ('-short', '-flat', '-secrets'): continue try: data = config.walk(key, key_types=key_types) result[key] = self._maybe_all(list_all, data, key_types, recurse, sanitize) except AccessError: access_denied = True invalid.append(key) except KeyError: invalid.append(key) if invalid: return self._error(_('Invalid keys'), result=result, info={ 'keys': invalid, 'key_types': key_types, 'access_denied': access_denied }) else: return self._success(_('Displayed settings'), result=result) class ConfigureMailboxes(Command): """ Add one or more mailboxes. If not account is specified, the mailbox is only assigned an ID for use in the metadata index. If an account is specified, the mailbox will be assigned to that account and configured for automatic indexing. """ SYNOPSIS = ('A', 'add', 'settings/mailbox', '[+] [--