Repository: nicolas33/offlineimap Branch: master Commit: ba4ecea9e4e4 Files: 171 Total size: 2.6 MB Directory structure: gitextract_hxxddv1_/ ├── .coveragerc ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE.md │ └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.rst ├── COPYING ├── Changelog.maint.md ├── Changelog.md ├── MAINTAINERS.rst ├── MANIFEST.in ├── Makefile ├── README.md ├── TODO.rst ├── bin/ │ └── offlineimap ├── contrib/ │ ├── README.md │ ├── helpers.py │ ├── internet-urllib3.py │ ├── release.py │ ├── release.sh │ ├── store-pw-with-gpg/ │ │ ├── README.md │ │ ├── gpg-pw.py │ │ ├── offlineimaprc.sample │ │ └── passwords-gmail.txt │ ├── systemd/ │ │ ├── README.md │ │ ├── offlineimap-oneshot.service │ │ ├── offlineimap-oneshot.timer │ │ ├── offlineimap-oneshot@.service │ │ ├── offlineimap-oneshot@.timer │ │ ├── offlineimap.service │ │ └── offlineimap@.service │ ├── tested-by.py │ └── upcoming.py ├── docs/ │ ├── Makefile │ ├── build-uploads.sh │ ├── doc-src/ │ │ ├── API.rst │ │ ├── conf.py │ │ ├── dco.rst │ │ ├── index.rst │ │ ├── repository.rst │ │ └── ui.rst │ ├── offlineimap.known_issues.txt │ ├── offlineimap.txt │ ├── offlineimapui.txt │ ├── rfcs/ │ │ ├── README.md │ │ ├── rfc1731.IMAP4_auth.txt │ │ ├── rfc1732.compatibiliy_IMAP2-IMAP2bis.txt │ │ ├── rfc1733.models_in_IMAP4.txt │ │ ├── rfc1734.POP3_AUTHentication │ │ ├── rfc2061.compatibility_IMAP4-IMAP2bis.txt │ │ ├── rfc2086.IMAP4_ACL_extension.txt │ │ ├── rfc2087.IMAP4_QUOTA_extension.txt │ │ ├── rfc2088.IMAP4_non_synchronizing_literals.txt │ │ ├── rfc2095.IMAP-POP_AUTHorize_extension.txt │ │ ├── rfc2177.IMAP4_IDLE_command.txt │ │ ├── rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt │ │ ├── rfc2192.IMAP_URL_scheme.txt │ │ ├── rfc2193.IMAP4_Mailbox_referrals.txt │ │ ├── rfc2195.IMAP-POP_AUTHorize_extension.txt │ │ ├── rfc2221.IMAP4_Login_referrals.txt │ │ ├── rfc2244.ACAP.txt │ │ ├── rfc2342.IMAP4_Namespace.txt │ │ ├── rfc2359.IMAP4_UIDPLUS_extension.txt │ │ ├── rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt │ │ ├── rfc2683.IMAP4_Implementation_recommendations.txt │ │ ├── rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt │ │ ├── rfc2971.IMAP4_ID_extension.txt │ │ ├── rfc3028.Sieve_Mail_filtering_language.txt │ │ ├── rfc3348.IMAP4_Child_Mailbox_extension.txt │ │ ├── rfc3501.IMAP4rev1.txt │ │ ├── rfc3502.MULTIAPPEND_extension.txt │ │ ├── rfc3503.Message_Disposition_Notification.txt │ │ ├── rfc3516.IMAP4_Binary_content_extension.txt │ │ ├── rfc3691.IMAP_UNSELECT_command.txt │ │ ├── rfc4314.IMAP4_ACL_extension.txt │ │ ├── rfc4315.IMAP_UIDPLUS_extension.txt │ │ ├── rfc4466.Collected_extensions_to_IMAP4_ABNF.txt │ │ ├── rfc4467.IMAP_URLAUTH_extension.txt │ │ ├── rfc4469.IMAP_CATENATE_extension.txt │ │ ├── rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt │ │ ├── rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt │ │ ├── rfc4731.IMAP4_Extension_to_SEARCH_command.txt │ │ ├── rfc4978.IMAP_Compress_extension.txt │ │ ├── rfc5032.IMAP_WITHIN_Search_extension.txt │ │ ├── rfc5161.IMAP_ENABLE_extension.txt │ │ ├── rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt │ │ ├── rfc5182.IMAP_extension_last_SEARCH_result.txt │ │ ├── rfc5182.Sieve_and_extensions.txt │ │ ├── rfc5255.IMAP_i18n.txt │ │ ├── rfc5257.IMAP_ANNOTATE_extension.txt │ │ ├── rfc5258.IMAP4_LIST_command_extension.txt │ │ ├── rfc5423.IM_Store_Events.txt │ │ ├── rfc5464.IMAP_METADATA_extension.txt │ │ ├── rfc5465.IMAP_NOTIFY_extension.txt │ │ ├── rfc5530.IMAP_Response_codes.txt │ │ ├── rfc5738.IMAP_UTF8.txt │ │ ├── rfc5788.IMAP4_Keyword_registry.txt │ │ ├── rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt │ │ ├── rfc5957.IMAP4_SORT_extension.txt │ │ ├── rfc6154.IMAP_LIST_Special-use_Mailboxes.txt │ │ ├── rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt │ │ ├── rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt │ │ └── rfc6331.Moving_Digest-MD5_to_Historic │ └── website-doc.sh ├── offlineimap/ │ ├── CustomConfig.py │ ├── __init__.py │ ├── accounts.py │ ├── bundled_imaplib2.py │ ├── emailutil.py │ ├── error.py │ ├── folder/ │ │ ├── Base.py │ │ ├── Gmail.py │ │ ├── GmailMaildir.py │ │ ├── IMAP.py │ │ ├── LocalStatus.py │ │ ├── LocalStatusSQLite.py │ │ ├── Maildir.py │ │ ├── UIDMaps.py │ │ └── __init__.py │ ├── globals.py │ ├── imaplibutil.py │ ├── imapserver.py │ ├── imaputil.py │ ├── init.py │ ├── localeval.py │ ├── mbnames.py │ ├── repository/ │ │ ├── Base.py │ │ ├── Gmail.py │ │ ├── GmailMaildir.py │ │ ├── IMAP.py │ │ ├── LocalStatus.py │ │ ├── Maildir.py │ │ └── __init__.py │ ├── threadutil.py │ ├── ui/ │ │ ├── Curses.py │ │ ├── Machine.py │ │ ├── Noninteractive.py │ │ ├── TTY.py │ │ ├── UIBase.py │ │ ├── __init__.py │ │ └── debuglock.py │ ├── utils/ │ │ ├── __init__.py │ │ ├── const.py │ │ ├── distro.py │ │ └── stacktrace.py │ └── virtual_imaplib2.py ├── offlineimap.conf ├── offlineimap.conf.minimal ├── offlineimap.py ├── requirements.txt ├── scripts/ │ └── get-repository.sh ├── setup.cfg ├── setup.py ├── snapcraft.yaml ├── test/ │ ├── .gitignore │ ├── OLItest/ │ │ ├── TestRunner.py │ │ ├── __init__.py │ │ └── globals.py │ ├── README │ ├── __init__.py │ ├── credentials.conf.sample │ └── tests/ │ ├── __init__.py │ ├── test_00_globals.py │ ├── test_00_imaputil.py │ ├── test_01_basic.py │ └── test_02_MappedIMAP.py └── tests/ ├── .gitignore ├── create_conf_file.py └── requirements.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] branch = True source = offlineimap [report] exclude_lines = if self.debug: pragma: no cover raise NotImplementedError if __name__ == .__main__.: ignore_errors = True omit = tests/* test/* ================================================ FILE: .github/CODEOWNERS ================================================ # This is a comment. # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. # Unless a later match takes precedence, @global-owner1 and @global-owner2 # will be requested for review when someone opens a pull request. #* @global-owner1 @global-owner2 # Order is important; the last matching pattern takes the most precedence. # When someone opens a pull request that only modifies JS files, only @js-owner # and not the global owner(s) will be requested for a review. #*.js @js-owner # You can also use email addresses if you prefer. They'll be used to look up # users just like we do for commit author emails. #docs/* docs@example.com * @chris001 ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ #### General informations - system/distribution (with version): - offlineimap version (`offlineimap -V`): - Python version: - server name or domain: - CLI options: #### Configuration file offlineimaprc ``` REMOVE PRIVATE DATA. ``` #### pythonfile (if any) ``` REMOVE PRIVATE DATA. ``` #### Logs, error ``` REMOVE PRIVATE DATA. ``` #### Steps to reproduce the error - - ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ > This v1.1 template stands in `.github/`. ### This PR > Add character x `[x]`. - [ ] I've read the [DCO](http://www.offlineimap.org/doc/dco.html). - [ ] I've read the [Coding Guidelines](http://www.offlineimap.org/doc/CodingGuidelines.html) - [ ] The relevant informations about the changes stands in the commit message, not here in the message of the pull request. - [ ] Code changes follow the style of the files they change. - [ ] Code is tested (provide details). ### References - Issue #no_space ### Additional information ================================================ FILE: .gitignore ================================================ # Backups. .*.swp .*.swo *~ # websites. /website/ /wiki/ # Generated files. *.html *.css /docs/dev-doc/ /build/ *.pyc offlineimap.1 offlineimapui.7 # Editors/IDEs tags # Generated conf files for Travis-CI tests. oli-travis.conf ================================================ FILE: .travis.yml ================================================ language: python python: - '2.7' notifications: webhooks: urls: - https://webhooks.gitter.im/e/975e807e0314c9fa189c on_success: always # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always on_start: never os: - linux env: global: - secure: jehlvkFxQbkvr73A0z3HGNC/knZQPKcaXLf6nByGpNE0ZTQKF7Y5KkNfeTcw4st7L7KuRZ1S/1bFtpMXTaplE6G0OtIEC4//SM+z+Dnadn2OY6wHiaapwZmmqDC5qVvcXPdmz/wTRsdrJSGLb2l6kEb91vRGbCCfHHf6Z2cF71U= - secure: kWdmWAFK4qrA73ONz1X8CJdHSER3bCBXjLfYHYEEMPCZep21bTITUXIfZBlSNN1888SQtYksuloRJmvj7xiY/hf/4lyWiqM3RgWQ+YptJMVOQX+Gara6vm4nGntKQwaXgZF2YHSh+NYwQm1VY6m0n1ye/vfOIJnYfgGTk5qAZYU= - secure: MzytYRX6HxgBj6Q3efkACTtDed8ZYO+P6UJrDA9IDtvffi8fAFb+wkQtKJrdcvMXNOap6fPe4c0EVGjgL5hFxmgC8yAh5t2YK7OhstAtq0ptKFlOcU24/drrkqoq040sAM/4Lc0nQCvYpz7bH370jzZl69rpbQWttwQR0i1e3Gw= - secure: RWvIOHSiv2kt6cfZR7MEueiAmC61bWMXAtgsC6gKq1u3BfENfqSBTA/heIy+nlu7AXK1b6hPMZDCHWK09Zz6Klkd9xZ1gkE/AARWseoo9UWgGjmfvqng1S6qpESeX2GnZGR9CuBXTPGhtbYLgtNlxAo+6uZLolz2utW2XNk3Z/Y= - secure: spivQv+vSJhE+ttn/Z6tANaINqiMSaJSucRqtoXR7PtioVDTOTmmL01Ja6dXuo8Ua5iVFtpZPDzqVpntQLKtjcywSK2zWnC9qbZYDfENr1/yIvfbSRjGeseq0eoY+fFp67FGZV4mIasdC3LOB0lRGOyrsX787fNKVQ8ZH0CRz0o= - secure: ZcY0TvTQnRCdoFkdbJPfDJJNx91tViwbpiOBkxNEa3u0RN48xkZkii35kNVBaEcVZHcT9C81ctHk4QX+plBkCsoj5GDf25scgcv1j9R9UoN/rIkmyTu1Znmc+3UQ2J+EnGLWVn5xJ7yT/l9NZeLfNbULQRjttwT4j2MBGxezgdM= matrix: - OUTLOOK_AUTH=PLAIN GMAIL_AUTH=XOAUTH2 - OUTLOOK_AUTH=LOGIN GMAIL_AUTH=XOAUTH2 matrix: include: - os: osx language: generic env: PYTHON=2.7.14 OUTLOOK_AUTH=PLAIN GMAIL_AUTH=XOAUTH2 - os: osx language: generic env: PYTHON=2.7.14 OUTLOOK_AUTH=LOGIN GMAIL_AUTH=XOAUTH2 allow_failures: - os: osx cache: pip before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update && brew install openssl readline; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export OSX_BREW_SSLCACERTFILE="/usr/local/etc/openssl/cert.pem"; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew outdated pyenv || brew upgrade pyenv; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install pyenv-virtualenv; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv install $PYTHON; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PYENV_VERSION="${PYTHON}"; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/Users/travis/.pyenv/shims:${PATH}"; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv virtualenv $PYTHON myvenv; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv versions; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python --version; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv version; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python --version; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m pip install -U pip; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m easy_install -U setuptools; fi install: - pip install -r requirements.txt - pip install -r tests/requirements.txt - export PATH=$PATH:. - python tests/create_conf_file.py script: - "./offlineimap.py -c ./oli-travis.conf" - codecov ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Realistic Code of Conduct 1. We mostly care about making our softwares better. 2. Everybody is free to decide how to contribute. 3. We believe in free speech. Everyone's entitled to their opinion. 4. Feel offended? This might be very well-deserved. 5. We don't need a code of conduct imposed on us, thanks. 6. Ignoring this Realistic Code of Conduct is welcome. ================================================ FILE: CONTRIBUTING.rst ================================================ .. -*- coding: utf-8 -*- .. vim: spelllang=en ts=2 expandtab: .. _OfflineIMAP: https://github.com/OfflineIMAP/offlineimap .. _Github: https://github.com/OfflineIMAP/offlineimap .. _repository: git://github.com/OfflineIMAP/offlineimap.git .. _maintainers: https://github.com/OfflineIMAP/offlineimap/blob/next/MAINTAINERS.rst .. _mailing list: http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project .. _Developer's Certificate of Origin: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/doc-src/dco.rst .. _Community's website: http://www.offlineimap.org .. _APIs in OfflineIMAP: http://www.offlineimap.org/documentation.html#available-apis .. _documentation: http://www.offlineimap.org/documentation.html .. _Coding Guidelines: http://www.offlineimap.org/doc/CodingGuidelines.html .. _Know the status of your patches: http://www.offlineimap.org/doc/GitAdvanced.html#know-the-status-of-your-patch-after-submission .. _How to fix a bug in open source software: https://opensource.com/life/16/8/how-get-bugs-fixed-open-source-software ================= HOW TO CONTRIBUTE ================= You'll find here the **basics** to contribute to OfflineIMAP_, addressed to users as well as learning or experienced developers to quickly provide contributions. **For more detailed documentation, see the** `Community's website`_. .. contents:: :depth: 3 Submit issues ============= Issues are welcome to both Github_ and the `mailing list`_, at your own convenience. Provide the following information: - system/distribution (with version) - offlineimap version (`offlineimap -V`) - Python version - server name or domain - CLI options - Configuration file (offlineimaprc) - pythonfile (if any) - Logs, error - Steps to reproduce the error Worth the read: `How to fix a bug in open source software`_. You might help closing some issues, too. :-) For the imaptients ================== - `Coding Guidelines`_ - `APIs in OfflineIMAP`_ - `Know the status of your patches`_ after submission - All the `documentation`_ Community ========= All contributors to OfflineIMAP_ are benevolent volunteers. This makes hacking to OfflineIMAP_ **fun and open**. Thanks to Python, almost every developer can quickly become productive. Students and novices are welcome. Third-parties patches are essential and proved to be a wonderful source of changes for both fixes and new features. OfflineIMAP_ is entirely written in Python, works on IMAP and source code is tracked with Git. *It is expected that most contributors don't have skills to all of these areas.* That's why the best thing you could do for you, is to ask us about any difficulty or question raising in your mind. We actually do our best to help new comers. **We've all started like this.** - The official repository_ is maintained by the core team maintainers_. - The `mailing list`_ is where all the exciting things happen. Getting started =============== Occasional contributors ----------------------- * Clone the official repository_. Regular contributors -------------------- * Create an account and login to Github. * Fork the official repository_. * Clone your own fork to your local workspace. * Add a reference to your fork (once):: $ git remote add myfork https://github.com//offlineimap.git * Regularly fetch the changes applied by the maintainers:: $ git fetch origin $ git checkout master $ git merge offlineimap/master $ git checkout next $ git merge offlineimap/next Making changes (all contributors) --------------------------------- 1. Create your own topic branch off of ``next`` (recently updated) via:: $ git checkout -b my_topic next 2. Check for unnecessary whitespaces with ``git diff --check`` before committing. 3. Commit your changes into logical/atomic commits. **Sign-off your work** to confirm you agree with the `Developer's Certificate of Origin`_. 4. Write a good *commit message* about **WHY** this patch (take samples from the ``git log``). Learn more ========== There is already a lot of documentation. Here's where you might want to look first: - The directory ``offlineimap/docs`` has all kind of additional documentation (man pages, RFCs). - The file ``offlineimap.conf`` allows to know all the supported features. - The file ``TODO.rst`` express code changes we'd like and current *Work In Progress* (WIP). ================================================ FILE: COPYING ================================================ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. ---------------------------------------------------------------- In addition, as a special exception, the copyright holders give permission to link the code of portions of this program with the OpenSSL library under certain conditions as described in each individual source file, and distribute linked combinations including the two. You must obey the GNU General Public License in all respects for all of the code used other than OpenSSL. If you modify file(s) with this exception, you may extend this exception to your version of the file(s), but you are not obligated to do so. If you do not wish to do so, delete this exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. ================================================ FILE: Changelog.maint.md ================================================ --- layout: page title: Changelog of the stable branch --- * The following excerpt is only usefull when rendered in the website. {:toc} This is the Changelog of the maintenance branch. **NOTE FROM THE MAINTAINER:** This branch comes almost as-is. With no URGENT requirements to update this branch (e.g. big security fix), it is left behind. If anyone volunteers to maintain it and backport patches, let us know! ### OfflineIMAP v6.7.0.3 (2016-07-26) #### Bug Fixes * sqlite: properly serialize operations on the database files ### OfflineIMAP v6.7.0.2 (2016-07-22) #### Bug Fixes * sqlite: close the database when no more threads need connection. ### OfflineIMAP v6.7.0.1 (2016-06-08) #### Bug Fixes * Correctly open and close sqlite databases. ### OfflineIMAP v6.3.2.1 (2011-03-23) #### Bug Fixes * Sanity checks for SSL cacertfile configuration. * Fix regression (UIBase is no more). * Make profiling mode really enforce single-threading. ================================================ FILE: Changelog.md ================================================ --- layout: page title: Changelog of mainline --- * The following excerpt is only usefull when rendered in the website. {:toc} ### OfflineIMAP v7.3.0 (2019-08-19) #### Notes Here comes a new release. The upstream imaplib2 project is discontinued. That's why I've decided to take over the maintenance of imaplib2 for offlineimap. For the use of offlineimap I've applied the pending PRs from imaplib2. I have applied another change sent to offlineimap (see OfflineIMAP/offlineimap#623). However, there are 3 important limitations: - I intend to maintain imaplib2 for offlineimap only. Everything will take place in the offlineimap project. I'll neither package imaplib2 nor maintain any "official" repository dedicated to imaplib2. If you want imaplib2 but not offlineimap, you should extract the file `offlineimap/bundled_imaplib2.py` from the offlineimap repository. Please, send your patches for imaplib2.py to the offlineimap project directly. - Starting from imaplib2 v2.100 (tagged: imaplib2-v2.100) I'm taking the patches in the lazy mode. This means that I won't make deep checks/tests of the changes. Hence, the quality and the stability might become a bit more fluctuating. For more stability, you might like to only consider the imaplib2 versions released with the stable versions of offlineimap. Don't expect changelogs dedicated to imaplib2. They will be part of the offlineimap changelogs. - All of this only applies to the py2 version of imaplib2. Sadly, offlineimap has few chances to be ported on py3 so I don't aim to maintain the py3 version of imaplib2. In this release, offlineimap is learning Happy Eyeballs. This release was tested by: - Nicolas Sebrecht #### Authors - Nicolas Sebrecht (4) - Ben Cotterell (1) - Dario Maiocchi (1) - Ilias Tsitsimpis (1) - Julien Cristau (1) - Olivier Mehani (1) #### Features - Implement Happy Eyeballs. [Olivier Mehani] - imaplib2 v2.101. [Nicolas Sebrecht] - imaplib2 v2.100. [Nicolas Sebrecht] #### Changes - Update readme to give an hint about Linux distros. [Dario Maiocchi] - travis: remove python3.6. [Nicolas Sebrecht] - README: add required dependency to rfc6555. [Nicolas Sebrecht] #### imaplib2 - Do not use TIMEOUT_MAX for Condition.wait(). [Ilias Tsitsimpis] - Use SSLContext if available so we send SNI. [Julien Cristau] - Don't expect trailing space on command completion. [Ben Cotterell] ### OfflineIMAP v7.2.4 (2019-06-08) #### Notes This release introduces mkdir -p alike folder creation and fixes cygwin support in Windows. This release was tested by: - Nicolas Sebrecht #### Authors - Nicolas Sebrecht (4) - kimim (2) - Jelmer Vernooij (1) - Kyle Altendorf (1) #### Features - mkdir -p alike folder creation. [Kyle Altendorf] #### Fixes - Use portable locker to support cygwin in Windows. [kimim] - contrib/release.py: don't break if sphinx-build is missing. [Nicolas Sebrecht] #### Changes - Update FSF postal address. [Jelmer Vernooij] - repository/IMAP: update copyright header date. [Nicolas Sebrecht] - PULL_REQUEST_TEMPLATE: add space between brackets to enable the edition in the gui. [Nicolas Sebrecht] ### OfflineIMAP v7.2.3 (2019-02-17) #### Notes A tiny release for one minor bug fix. This release was tested by: - Nicolas Sebrecht #### Authors - Mart Lubbers (1) #### Fixes - add checks in curses ui for small windows. [Mart Lubbers] ### OfflineIMAP v7.2.2 (2018-12-22) #### Notes With this release offlineimap can renew the token for OAUTH2. There is better integration for ArchLinux and OSX. SSL configuration options are more consistent. There are bug fixes about maxage and GSSAPI. The imaplib2 library looks discontinued. I wonder we'll have no other choice than maintaining our own fork. This release was tested by: - Nicolas Sebrecht #### Authors - Nicolas Sebrecht (5) - Philippe Loctaux (4) - Benedikt Heine (2) - Carnë Draug (2) - Frode Aannevik (1) - Robbie Harwood (1) #### Features - 2890dec Added ssl certfile on osx for openssl pacakge on homebrew. [Philippe Loctaux] - 761e10e Add Archlinux to list of supported distros. [Philippe Loctaux] #### Fixes - 8692799 Fix expired oauth2_access_token. [Frode Aannevik] - 096aa07 Handle empty token with complete GSSAPI context. [Robbie Harwood] - a51064e maxage: always compute the remote cache list for min_uid. [Nicolas Sebrecht] - 698ec64 offlineimap.conf: minor fixes. [Nicolas Sebrecht] - af3a35a offlineimap/utilis/distro.py: indentation fix. [Philippe Loctaux] - d3ba837 Fix typo in exception message. [Benedikt Heine] - c9005cd Check if username is provided before trying plain authentication.. [Carnë Draug] #### Changes - 5f9474e Print username instead of accountname when asking for password. [Carnë Draug] - ce9a198 Chain tls_level and ssl_version only if ssl is enabled. [Benedikt Heine] - 6ef5937 docs/website-doc.sh: minor improvements in comments of versions.yml. [Nicolas Sebrecht] - 4544bb1 contrib/release.py: minor UI improvement. [Nicolas Sebrecht] - d930125 fix dates in copyright lines. [Nicolas Sebrecht] ### OfflineIMAP v7.2.1 (2018-06-16) #### Notes This new version introduces interesting features. The fingerprints now accepts hashes in sha224, sha256, sha384 and sha512 to improve the compatibility with IMAP servers. There's a new script in ./contrib to store passwords with GPG. The new GSSAPI library for kerberos gets a fix about authentication. Gmail labels can now have parenthesis and the hostname can have path separators in theirs names. There's a lot of other minors improvements to make offlineimap better (in the documentation, UI, configuration file and the code). This release was tested by: - Nicolas Sebrecht Thanks to all the contributors. A lot of patches are first time contributions to this project. This is very pleasant. Special thanks to Ilias Tsitsimpis, Eygene Ryabinkin, Chris Coleman our long time contributors involved in this release and Sebastian Spaeth who is still paying for the domain name! #### Authors - Nicolas Sebrecht (9) - velleto (6) - Chris Coleman (1) - Edgar HIPP (1) - Eygene Ryabinkin (1) - Lorenzo (1) - Michael Billington (1) - Robbie Harwood (1) #### Features - Script to store passwords in a file with GPG or using OSX's secure keychain. [Lorenzo] - Added support for sha512, sha384, sha256, sha224 hashing algorithms to calculate server certificate fingerprints.. [velleto] #### Fixes - Pass username through in GSSAPI connections. [Robbie Harwood] - Gmail: allow parenthesis in labels. [Nicolas Sebrecht] - Correct typographical errors in offlineimap.conf. [Michael Billington] - Create filenames with no path separators in them. [Eygene Ryabinkin] #### Changes - imapserver: fix copyright line. [Nicolas Sebrecht] - Available hashes added to documentation.. [velleto] - Documented the now allowed use of colon separated fingerprints with examples.. [velleto] - Allow users to keep colons between each hex pair of server certificate fingerprint in configuration file.. [velleto] - Removed uneccessary call of list() on zip() object.. [velleto] - Changed the 'exception raised' message, to be more understandable.. [velleto] - Make CTRL-C message more clear. [Edgar HIPP] - setup: add long_description. [Nicolas Sebrecht] - offlineimap.conf: fix comment about gssapi. [Nicolas Sebrecht] - Add self to maintainers. Update email address.. [Chris Coleman] - Makefile: targz: don't set the abbrev in the archive directory name. [Nicolas Sebrecht] - contrib: learn to build website/_uploads. [Nicolas Sebrecht] - docs/website-doc.sh: limit the number of exported versions in _data/announces.yml. [Nicolas Sebrecht] - Makefile: targz: update files. [Nicolas Sebrecht] - Makefile: clean: remove __pycache__ directories. [Nicolas Sebrecht] ### OfflineIMAP v7.2.0 (2018-04-07) #### Notes The biggest change with this release is the introduction of automated tests; thanks to Chris from http://www.espacenetworks.com. Robbie Hardwood from RedHat switched the GSSAPI dependency from pykerberos to python-gssapi because it's more active and has more pleasant interface. The shebang is fixed back to python2 to fix issues on some environments. The UI was improved to show both the local and remote foldernames (usefull when nametrans is enabled). Thanks to all the contributors. This release was tested by: - Nicolas Sebrecht - Remi Locherer #### Authors - Nicolas Sebrecht (9) - Musashi69 (1) - Robbie Harwood (1) - chris001 (1) #### Features - Autmomated testing using Travis and CodeCov.io!. [chris001] - README: travis: add badge for the next branch. [Nicolas Sebrecht] - travis: add notification to gitter room OfflineIMAP/offlineimap. [Nicolas Sebrecht] #### Fixes - offlineimap.py: fix shebang to python2. [Nicolas Sebrecht] - bin/offlineimap: fix shebang to env python2. [Nicolas Sebrecht] #### Changes - Port to python-gssapi from pykerberos. [Robbie Harwood] - requirements: add gssapi as optional dependency. [Nicolas Sebrecht] - make UI output show local AND remote dirs involved. [Musashi69] - maxsyncaccounts: improve documentation. [Nicolas Sebrecht] ### OfflineIMAP v7.1.5 (2018-01-13) #### Notes This minor release fixes a bug about maxage failing to upload some emails. Also, this introduces the snapcraft.yaml to package offlineimap with this packaging system. This release was tested by: - Nicolas Sebrecht - Remi Locherer #### Authors - Nicolas Sebrecht (4) - Evan Dandrea (1) - John Ferlito (1) #### Features - Initial commit of snapcraft.yaml. [Evan Dandrea] #### Fixes - maxage: don't consider negative UIDs when computing min UID. [Nicolas Sebrecht] - Add missing space to output string. [John Ferlito] #### Changes - folder: IMAP: improve search logging. [Nicolas Sebrecht] - no UIDPLUS: improve logging on failures. [Nicolas Sebrecht] - github: remove the trick to download the PR. [Nicolas Sebrecht] ### OfflineIMAP v7.1.4 (2017-10-29) #### Notes Here is a bugfix release for v7.1.3. Two regressions got fixes and the --delete-folder CLI option now expects an UTF-8 folder name when utf8foldernames is enabled. This release was tested by: - Nicolas Sebrecht #### Authors - Nicolas Sebrecht (5) - Thomas Merkel (1) #### Fixes - utf8foldernames: fix missing decode argument. [Nicolas Sebrecht] - Fix: if any tunnel (preauth_tunnel or transport_tunnel) the hostname should not be required. [Thomas Merkel] #### Changes - utf8foldernames: support --delete-folder with UTF-8 folder name. [Nicolas Sebrecht] - contrib/release.py improvements ### OfflineIMAP v7.1.3 (2017-10-08) #### Notes This release introduces a new experimental utf8foldernames configuration option. We already had the "tricky" decodefoldernames which is now deprecated. The new code is the correct implementation for this feature. The changes are neat and rather small. All the users having decodefoldernames are requested to move to utf8foldernames. This requires to update almost all the functions like nametrans, folderfilter, etc, because they work on the UTF-8 encoding. See the documentation for more. Thank you Urs Liska for this contribution! In the long run, the idea is to: 1. Remove decodefoldernames in favour of utf8foldernames. 2. Promote utf8foldernames up to stable. 3. Turn utf8foldernames on by default. Currently, folders with non-ASCII characters in their name have to be fully re-downloaded. So, there's a bit more work to be done to have (3) and maybe (2). Also, this release includes a fix about remotehost and transporttunnel that would require some testing. Thanks Thomas Merkel! There are documentation improvements, improved errors and minor code cleanups, too. This release was tested by: - Nicolas Sebrecht - Remi Locherer #### Authors - Nicolas Sebrecht (11) - Urs Liska (8) - Thomas Merkel (1) #### Features - utf8: implement utf8foldernames option. [Urs Liska] - utf8: document new feature, deprecate old one. [Urs Liska] #### Fixes - remotehost should not be required if transporttunnel is used. [Thomas Merkel] - accounts: error out when no folder to sync. [Nicolas Sebrecht] - sqlite: provide better message error for insert. [Nicolas Sebrecht] - folder: Gmail: fix copyright header. [Nicolas Sebrecht] #### Changes - man: remove mention of experimental support for python 3. [Nicolas Sebrecht] - man: mention the supported directions of the syncs. [Nicolas Sebrecht] - folder: Gmail: remove dead code. [Nicolas Sebrecht] - upcoming.py: get header template from external file. [Nicolas Sebrecht] - upcoming.py: display a message with the filename once written. [Nicolas Sebrecht] - contrib/helpers: sort testers by name. [Nicolas Sebrecht] - Remove some unnecessary whitespace (in existing code). [Urs Liska] - MAINTAINERS: Rainer is not currently active. [Nicolas Sebrecht] ### OfflineIMAP v7.1.2 (2017-07-10) #### Notes This release introduces better Davmail support, better reliability when in IMAP/IMAP mode, better output on some errors, and minor fixes. The provided systemd files are improved. The imaplib2 requirement is now v2.57. Remi Locherer is joining our tester team. Great! Starting with this release, the feedbacks from the testers are recorded in the release notes, the git logs and the Changelog. Thanks to all of them for improving the releases. This release was tested by: - benutzer193 - Nicolas Sebrecht - Remi Locherer #### Authors - Nicolas Sebrecht (20) - Hugo Osvaldo Barrera (5) - Alvaro Pereyra (1) - benutzer193 (1) #### Features - contrib/release.py: consider positive feedbacks from testers. [Nicolas Sebrecht] - Introduce the github CODEOWNERS file. [Nicolas Sebrecht] - IMAP/IMAP: continue to sync if the local side does not return a valid UID on upload. [Nicolas Sebrecht] #### Fixes - folder/IMAP: introduce dedicated parsing for davmail (not supporting UIDPLUS). [Nicolas Sebrecht] - offlineimap.conf: minor typo fix. [Alvaro Pereyra] - Respect systemd conventions for timers. [Hugo Osvaldo Barrera] - Use a pre-existing target for systemd services. [Hugo Osvaldo Barrera] - Remove invalid systemd setting. [Hugo Osvaldo Barrera] - systemd: remove unused watchdog functionality. [benutzer193] - gitignore generated css file. [Nicolas Sebrecht] - Changelog: fix syntax. [Nicolas Sebrecht] #### Changes - Increase imaplib2 requirement from v2.55 to v2.57. [Nicolas Sebrecht] - folder/IMAP: improve the warning when we can't parse the returned UID. [Nicolas Sebrecht] - Provide more details in error message when SSL fails on non-standard port. [Nicolas Sebrecht] - Use basic logger (since systemd picks up stdout). [Hugo Osvaldo Barrera] - Explain how to override systemd values. [Hugo Osvaldo Barrera] - systemd: add documentation entry in configuration files. [Nicolas Sebrecht] - offlineimap.conf: ssl must be disabled to force STARTTLS in some cases. [Nicolas Sebrecht] - Advise singlethreadperfolder when offlineimap hangs. [Nicolas Sebrecht] - offlineimap.conf: minor improvements. [Nicolas Sebrecht] - contrib: more release automation. [Nicolas Sebrecht] - MAINTAINERS: Remi Locherer joins the team of testers. [Nicolas Sebrecht] - systemd: README: credit Hugo as contributor. [Nicolas Sebrecht] ### OfflineIMAP v7.1.1 (2017-05-28) #### Notes This release has some interesting fixes, including one for the Blinkenlights UI. Otherwise, there is no big change since the previous version. Furthermore, this release was tested by: - Remi Locherer #### Authors - Nicolas Sebrecht (17) - Chris Coleman (1) - Ilias Tsitsimpis (1) - Maximilian Kaul (1) - benutzer193 (1) - Ævar Arnfjörð Bjarmason (1) #### Features - contrib: introduce a tool to produce the "upcoming notes". [Nicolas Sebrecht] - contrib: secure HTTPS test internet is connected.. [Chris Coleman] - Env info (used by -V and banner): add openssl version. [Nicolas Sebrecht] - docs: learn to build html files for the manual pages. [Nicolas Sebrecht] #### Fixes - Acquire lock before updating the CursesLogHandler window. [Ilias Tsitsimpis] - maxage: use the remote folder first to compute min_uid. [Nicolas Sebrecht] - Fix systemd.timer: initialize timer after boot. [benutzer193] - XOAUTH2: don't try this authentication method when not configured. [Nicolas Sebrecht] - mbnames: don't duplicate entries in autorefresh mode. [Nicolas Sebrecht] - docs: update the instructions for creating OAuth projects for GMail. [Ævar Arnfjörð Bjarmason] - Fixed typo in doc: tls_1_2 => tls1_2. [Maximilian Kaul] - IMAP: UIDPLUS: correctly warn about weird responses from some servers. [Nicolas Sebrecht] - website-doc: force copy of the new HTML generated man pages. [Nicolas Sebrecht] - Makefile: fix clean target. [Nicolas Sebrecht] #### Changes - MAINTAINERS: benutzer193 joins the testers team. [Nicolas Sebrecht] - IMAP: UIDPLUS: improve error message on response error for new UID. [Nicolas Sebrecht] - Display the imaplib and python versions for each normal run. [Nicolas Sebrecht] - imapserver: provide some SSL info while in imap debug mode. [Nicolas Sebrecht] - manual: improve the documentation about sqlite migration. [Nicolas Sebrecht] - documentation: add entry for faulting folders with Microsoft servers. [Nicolas Sebrecht] - website-doc.sh: add hint on API removal. [Nicolas Sebrecht] - README: refactorize sections. [Nicolas Sebrecht] ### OfflineIMAP v7.1.0 (2017-04-16) #### Notes The most important change is the removal of the status_backend configuration option and that's why we're moving to v7.1.0. There are other small bug fixes and improvements. However, the codebase didn't change much since v7.0.14. #### Authors - Nicolas Sebrecht (6) - benutzer193 (4) - Ilias Tsitsimpis (1) #### Fixes - doc: Fix typo in offlineimap.1 man page. [Ilias Tsitsimpis] - README: we moved to imaplib2 v2.57. [Nicolas Sebrecht] - README: mark porting to py3 as stalled. [Nicolas Sebrecht] - folder: UIDMaps: ignore KeyError failure while removing keys. [Nicolas Sebrecht] #### Changes - Remove support for the status_backend configuration option. [Nicolas Sebrecht] - folder/IMAP: improve handling of "matchinguids" error while searching headers. [Nicolas Sebrecht] - Adjust README to systemd service file changes. [benutzer193] - Remove oneshot switch from systemd services. [benutzer193] - Use oneshot services for systemd timers. [benutzer193] - Create systemd oneshot services. [benutzer193] - website-doc.sh: versions.yml: set versions in order. [Nicolas Sebrecht] ### OfflineIMAP v7.0.14 (2017-03-11) #### Notes Here is a new small fixup release for the v7.0 series. The first v7.0.0 release is near to 8 months old. This v7.0.14 release is more reliable than v6.7.0.3. Hence, I'm deprecating the v6.7 series. Now, you are all enjoined to migrate to v7.0.14. Migrating back to v6.7 is not supported so you might like to backup your local maildirs and metadata first. We will fully remove the legacy text backend driver in near future. The SQLite driver proved to be better for both performance and reliability. With this release we use imaplib2 v2.57 to support some faulting IMAP servers, fix a little bug about the backend migration to SQLite and serialize the sync processes to prevent from issues when both IDLE and autorefresh are enabled. Happy sync'ing! #### Authors - Nicolas Sebrecht (5) - 927589452 (2) - Jens Heinrich (1) - Stéphane Graber (1) #### Fixes - SQLite: avoid concurrent writes on backend migration. [Nicolas Sebrecht] - Fix ipv6 configuration handling. [Stéphane Graber] - Prevent synchronization of identical folders from multiple threads. [Nicolas Sebrecht] #### Changes - Bump from imaplib2 v2.55 to v2.57. [Nicolas Sebrecht] - scripts/get-repository.sh: use portable /bin/sh. [Jens Heinrich] - MAINTAINERS: add new tester. [Nicolas Sebrecht] - scripts/get-repository.sh: use env to call bash. [mailinglists@927589452.de] ### OfflineIMAP v7.0.13 (2017-01-27) #### Notes Here is a small release with some new features. IMAP servers are better supported. The release cycle was improved. Now, we include a new freeze time before important releases. #### Authors - Nicolas Sebrecht (8) - lkcl (2) - Chris Smart (1) #### Features - init: register SIGABRT and handle as per SIGUSR2. [Chris Smart] - add documentation about SIGABRT. [Nicolas Sebrecht] - learn repository retrycount configuration option. [lkcl] - learn authproxy configuration option. [lkcl] #### Fixes - folder: IMAP: add missing whitespace in error message. [Nicolas Sebrecht] - repository: IMAP: correctly check the response while listing remote folders. [Nicolas Sebrecht] - release.sh: correctly sort releases to compute latest stable and rc. [Nicolas Sebrecht] #### Changes - manual: KNOWN ISSUES: add documentation about the deletions. [Nicolas Sebrecht] - folder: IMAP: improve error message when Dovecot returns any data for UID FETCH. [Nicolas Sebrecht] - MAINTAINERS: add new official testers. [Nicolas Sebrecht] ### OfflineIMAP v7.0.12 (2016-11-30) #### Notes Quick small release to fix v7.0.11 for the users of nametrans. #### Authors - Abdo Roig-Maranges (1) - Darshit Shah (1) - Nicolas Sebrecht (1) #### Features - Enable environment variable expansion on Repository.localfolders. [Darshit Shah] #### Fixes - repository: Base: fix typo in folder variable name. [Abdo Roig-Maranges] - MAINTAINERS: minor: fix rendering. [Nicolas Sebrecht] ### OfflineIMAP v7.0.11 (2016-11-30) #### Notes Very small release to fix a regression about structure comparison in v7.0.10. #### Authors - Nicolas Sebrecht (2) #### Fixes - repository: Base: fix folder structure comparison. [Nicolas Sebrecht] #### Changes - MAINTAINERS: add all the contributors. [Nicolas Sebrecht] ### OfflineIMAP v7.0.10 (2016-11-28) #### Notes This release is mainly about improving reliability. The biggest changes are about comparing the local and remote structures of folders. The Gmail repository type allows to tune some predefined options for advanced use cases. Offlineimap learns where to find the default OpenSUSE certificate. Some code refactoring and documentation improvements. #### Authors - Nicolas Sebrecht (15) - Stéphane Albert (4) - Abdo Roig-Maranges (2) - Xudong Zhang (1) - altruizine (1) - Ævar Arnfjörð Bjarmason (1) #### Features - GMail: Add ability to set a custom host/port/ssl etc.. [Ævar Arnfjörð Bjarmason] - Add OpenSUSE to list of supported distros. [altruizine] #### Fixes - repository: Base: fix name of the status folder. [Abdo Roig-Maranges] - repository: Base: rework the structure folders comparison. [Nicolas Sebrecht] - Fix remaining instance of check_uid_validity refactoring. [Abdo Roig-Maranges] - Fix the profile mode. [Nicolas Sebrecht] - folder: Maildir: actually try to use Delivery-Date if Date is broken. [Nicolas Sebrecht] - Fix decodefoldernames not applying in folder sync. [Stéphane Albert] - Fix mbnames writing with folders using utf-8. [Stéphane Albert] - Fix utf7 decode error not caught. [Stéphane Albert] - Fix md5 folder generation wanting unicode. [Stéphane Albert] - Fix bug: should not compare list to int. [Xudong Zhang] #### Changes - folder: IMAP: display error message before starting next try. [Nicolas Sebrecht] - offlineimap.conf: XOAUTH2: certificate validation is required for Gmail. [Nicolas Sebrecht] - offlineimap.conf: autorefresh points to maxsyncaccounts. [Nicolas Sebrecht] - offlineimap.conf: use 'Offlineimap' to name the software. [Nicolas Sebrecht] - offlineimap.conf: add comments for the readonly configuration option. [Nicolas Sebrecht] - offlineimap.conf: mbnames: provide sample for the folderfilter option. [Nicolas Sebrecht] - Minor code refactoring. [Nicolas Sebrecht] - Don't allow negative values for autorefresh. [Nicolas Sebrecht] - Manual: add known issues entry about XOAUTH2 "invalid_grant". [Nicolas Sebrecht] - repository: Gmail: fix copyright line. [Nicolas Sebrecht] ### OfflineIMAP v7.0.9 (2016-10-29) #### Notes Let's go for this small but still interesting release. The Blinkenlights UI got fixed. Reliability for IMAP/IMAP setups is improved. The sqlite backend now honors the fsync configuration option. This allows commits to the database to be postponed. This might be usefull to disable the default fsync for some use cases like cache migration from text to sqlite, syncing after long away periods and more generally when a lot of new email entries must be written to the cache. Because of this change the old fsync option is marked EXPERIMENTAL. However, setups using the plain text cache are not concerned. Bear in mind that disabling fsync greatly decreases reliability when resuming from unexpected halts. Small code cleanups, too. #### Authors - Nicolas Sebrecht (4) - Giel van Schijndel (1) - Ilias Tsitsimpis (1) #### Features - SQLite: make postponing transaction committing possible.. [Giel van Schijndel] #### Fixes - UIDMaps: ensure we don't update the map file in dry run mode. [Nicolas Sebrecht] - UIDMaps: prevent from leaving a truncated map file. [Nicolas Sebrecht] - Fix flickering in Blinkenlights UI. [Ilias Tsitsimpis] #### Changes - UIDMaps: reorder imports. [Nicolas Sebrecht] - folder: IMAP: remove unused import. [Nicolas Sebrecht] ### OfflineIMAP v7.0.8 (2016-10-08) #### Notes Very small release to fix the broken UI relying on Curses. Thanks for the contributors! #### Authors - Nicolas Sebrecht (4) - Ilias Tsitsimpis (1) - Stéphane Albert (1) #### Features - Introduce contrib/README.md. [Nicolas Sebrecht] #### Fixes - Import ui before threadutil to resolve circular dependency. [Ilias Tsitsimpis] - Fix implicit call to unicode() from UI functions. [Stéphane Albert] #### Changes - imapserver: minor code cleaning: reorder methods. [Nicolas Sebrecht] - website-doc.sh: print usage when no argument is given. [Nicolas Sebrecht] - Changelog: add remark about singlethreadperfolder in the resume. [Nicolas Sebrecht] ### OfflineIMAP v7.0.7 (2016-09-21) #### Notes With this release, IDLE mode is a bit improved regarding stability. Offlineimap learns the default path to the certificate for Gentoo. The singlethreadperfolder configuration option is marked stable. There are few improvements for logs and documentation. Minor code refactoring, too. #### Authors - Nicolas Sebrecht (12) - Dan Loewenherz (1) - Espen Henriksen (1) #### Features - Add gentoo cert path for OS-DEFAULT. [Espen Henriksen] - Remove EXPERIMENTAL flag for the singlethreadperfolder configuration option. [Nicolas Sebrecht] #### Fixes - Ensure logs are in bytes for PLAIN authentication. [Nicolas Sebrecht] - Minor: utils: distro: fix copyright line. [Nicolas Sebrecht] - README: minor copy edits. [Dan Loewenherz] - IDLE: protect all calls to imapobj.noop() (coonection might be dropped). [Nicolas Sebrecht] - XOAUTH2: raise error if string 'error' is in the response. [Nicolas Sebrecht] #### Changes - Set singlethreadperfolder configuration option when in idle mode. [Nicolas Sebrecht] - repository: IMAP: cache the idle folders in memory. [Nicolas Sebrecht] - mbnames: add info output messages in dry run mode. [Nicolas Sebrecht] - mbnames: remove non-required argument. [Nicolas Sebrecht] - offlineimap.conf: explain hooks in idle mode. [Nicolas Sebrecht] - Explain how to submit issues in more files. [Nicolas Sebrecht] - README: explain the a2x dependency to build the man page. [Nicolas Sebrecht] ### OfflineIMAP v7.0.6 (2016-08-21) #### Notes Evaluated XOAUTH2 configuration options are fixed. With this release, offlineimap can try to keep the UIDs in order. #### Authors - Nicolas Sebrecht (10) - James E. Blair (2) #### Features - Learn singlethreadperfolder configuration option. [James E. Blair] - folder: Base: sort message UID list. [James E. Blair] #### Fixes - Maildir: add missing exception instance "as e" in except clause. [Nicolas Sebrecht] - XOAUTH2: fix evaluated configuration options. [Nicolas Sebrecht] #### Changes - XOAUTH2: improve error message while trying to get access token. [Nicolas Sebrecht] - Show python version for -V CLI option. [Nicolas Sebrecht] - README: link Python 3 version to issues. [Nicolas Sebrecht] - offlineimap.conf: add note about Gmail\All Mail keeping the emails while deleted. [Nicolas Sebrecht] - release.sh: minor enhancements. [Nicolas Sebrecht] ### OfflineIMAP v7.0.5 (2016-08-10) #### Notes Bugfix release. The machineui is fixed and the dry-run mode is a bit improved. Thanks to all the contributors and bug reporters. This release is yours. #### Authors - Nicolas Sebrecht (6) - Wieland Hoffmann (2) - Łukasz Żarnowiecki (2) - Christopher League (1) #### Fixes - don't delete messages in local cache in dry-run mode. [Nicolas Sebrecht] - Fix typo in format string in machineui. [Christopher League] #### Changes - folder: IMAP: change raw assert to OfflineImapError. [Nicolas Sebrecht] - folder: IMAP: add 'imap' debug output before calling FETCH. [Nicolas Sebrecht] - explicitly set __hash__ of Base class to None. [Łukasz Żarnowiecki] - imapserver: change lambdas with map to list comprehension. [Łukasz Żarnowiecki] - Clarify which settings are required for mbnames. [Wieland Hoffmann] - Remove an unused import. [Wieland Hoffmann] - folder: Base: minor style fix. [Nicolas Sebrecht] - CONTRIBUTING: add link to external page on "How to fix a bug". [Nicolas Sebrecht] - README: add link to the official repository on top of the page. [Nicolas Sebrecht] ### OfflineIMAP v7.0.4 (2016-08-02) #### Notes Small bugfix release for Gmail users. #### Authors - Nicolas Sebrecht (1) #### Fixes - ConfigHelperMixin must be new-style class to not break inheritance. [Nicolas Sebrecht] ### OfflineIMAP v7.0.3 (2016-07-30) #### Notes Here's a new bugfix release for the v7.0.x series. Only time we let us know if it's a good release. However, I'm more confident. Thanks for the reports and feedbacks! #### Authors - Nicolas Sebrecht (11) #### Fixes - Make systemd service kill offlineimap as expected. [Nicolas Sebrecht] - XOAUTH2: fix the \*\_eval configuration options. [Nicolas Sebrecht] - IMAP: don't take junk data for valid mail content. [Nicolas Sebrecht] - offlineimap.conf: allow non-spaces in the account list. [Nicolas Sebrecht] - Properly ignore folders with invalid characters (sep) in their name. [Nicolas Sebrecht] #### Changes - Add the repository name when connecting. [Nicolas Sebrecht] - Github template: add system/distribution. [Nicolas Sebrecht] - XOAUTH2: use one "public" attribute everywhere for self.oauth2_request_url. [Nicolas Sebrecht] - Code style and minor code enhancements. [Nicolas Sebrecht] - Manual: add known issue about netrc. [Nicolas Sebrecht] ### OfflineIMAP v7.0.2 (2016-07-27) #### Notes Small release to fix regression introduced in v7.0.0. #### Authors - Nicolas Sebrecht (1) - Philipp Meier (1) - Ævar Arnfjörð Bjarmason (1) #### Features - offlineimap.conf: learn to evaluate oauth2 related options. [Nicolas Sebrecht] #### Fixes - GmailMaildir: don't add a tuple to syncmessagesto_passes. [Philipp Meier] - Remove double import of "six". [Ævar Arnfjörð Bjarmason] ### OfflineIMAP v7.0.1 (2016-07-26) #### Notes This is a small stable release fixing all the reported regressions and issues about v7.0.0. #### Authors - Nicolas Sebrecht (9) #### Fixes - sqlite: properly serialize operations on the databases. [Nicolas Sebrecht] - IMAP/IMAP: fix import issue about UIDMaps. [Nicolas Sebrecht] - offlineimap.conf: allow non-spaces in the account list. [Nicolas Sebrecht] - website-doc.sh: fix link in announces.yml. [Nicolas Sebrecht] - release.sh: don't mess the mainline Changelog with commits in maint. [Nicolas Sebrecht] #### Changes - Improve error message when ssl_version must be set due to the tls_level. [Nicolas Sebrecht] - Code cleanups. - website-doc: order announces by date. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0 (2016-07-22) #### Notes Finally, the new v7.0.0 is ready. This comes with breaking changes: - Passwords are now expected in Unicode almost everywhere. They are used with the UTF-8 charset. However, some configuration options are not UTF-8 friendly mostly because of library limitations (e.g.: `remotepass`). Users with Unicode caracters in the passwords are recommended to use a file or `remotepasseval`. - The sqlite database is the default. Please, read [this blog post]({% post_url 2016-05-19-sqlite-becomes-default %}). - The PID file is no longer used because offlineimap is able to run multiple instances. Please read the intermediate changelogs. #### Authors - Nicolas Sebrecht (9) #### Features - release.sh: learn to merge maint branch into next before releasing. [Nicolas Sebrecht] #### Fixes - sqlite: close the database when no more threads need access. [Nicolas Sebrecht] - Fix attribute name _utime_from_header. [Nicolas Sebrecht] - Maildir: OfflineImapError is missing the severity argument. [Nicolas Sebrecht] - Fix: configparser does not know about python types like u"". [Nicolas Sebrecht] - Manual: offlineimapui: fix minor rendering issue. [Nicolas Sebrecht] #### Changes - --info: allow user to enter a password. [Nicolas Sebrecht] - Remove dead code: the description of the passes is never used. [Nicolas Sebrecht] - offlineimap.conf: improve documentation for copy_ignore_eval. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0-rc5 (2016-07-12) #### Notes This is a short -rc5 to stabilize the code with late improvements, mostly. #### Authors - Nicolas Sebrecht (9) - Ævar Arnfjörð Bjarmason (1) #### Features - learn --delete-folder CLI option. [Nicolas Sebrecht] #### Fixes - mbnames: fix the filename extension for the intermediate files. [Nicolas Sebrecht] - manual: offlineimap knows -V CLI option. [Nicolas Sebrecht] - manual: remove unkown --column CLI option. [Nicolas Sebrecht] - code of conduct: try to clarify what item 3 might mean. [Ævar Arnfjörð Bjarmason] #### Changes - mbnames: enable action at correct time. [Nicolas Sebrecht] - mbnames: output message on errors while reading intermediate files. [Nicolas Sebrecht] - --help: move -V option up. [Nicolas Sebrecht] - init: factorize code to get active accounts. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0-rc4 (2016-07-04) #### Notes Here we are to stabilize the code. I don't expect to merge features anymore. When emails failed to download, offlineimap was raising the same issues again and again. Users can now filter emails based on UID numbers. The mbnames was missing a way to remove obsolete entries from deleted accounts. Hence, --mbnames-prune is added. Syncing folders with the local "sep" characters in their names was causing troubles on next syncs. They are now filtered with a warning message. IMAP/IMAP mode is improved: this was suffuring a (rare) bug related to concurrent writes. Usual code cleanups and minor improvements are included in this release. I think this candidate is more stable than the previous v6.7.0 stable. Enjoy! #### Authors - Nicolas Sebrecht (17) #### Features - Learn to not download UIDs defined by the user. [Nicolas Sebrecht] - Learn --mbnames-prune CLI option. [Nicolas Sebrecht] #### Fixes - UIDMaps (IMAP/IMAP mode): correctly protect from concurrent writes. [Nicolas Sebrecht] - Correctly reraise errors with six. [Nicolas Sebrecht] - Don't sync folders with local separator characters in their names. [Nicolas Sebrecht] #### Changes - Minor: improve "Copy message" output. [Nicolas Sebrecht] - threadutil: use 'with' statements for lock. [Nicolas Sebrecht] - Code cleanups and minor improvements. [Nicolas Sebrecht] - release.sh: get_git_who(): remove unnecessary blank line. [Nicolas Sebrecht] - website-doc.sh: fix line continuation. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0-rc3 (2016-06-27) #### Notes The most important changes are: - The passwords (and usernames) are now expected in Unicode. - The sync_deletes feature is marked stable. - It is possible to disable STARTTLS if it is failing. - mbnames correctly honors the `-a` CLI option. #### Authors - Nicolas Sebrecht (31) - Ilias Tsitsimpis (1) #### Features - Learn to disable STARTTLS. [Nicolas Sebrecht] - Require usernames and passwords to be UTF-8 encoded. [Nicolas Sebrecht] - offlineimap.conf: sync_deletes option is stable. [Nicolas Sebrecht] - Learn -V CLI option. [Nicolas Sebrecht] #### Fixes - Don't try to copy messages with UID == 0. [Nicolas Sebrecht] - Avoid removing of data when user removed a maildir. [Nicolas Sebrecht] - When called with -a, mbnames must not erase entries of other accounts. [Nicolas Sebrecht] - GmailMaildir: quick mode is not compatible with utime_from_header. [Nicolas Sebrecht] - Manual: offlineimapui: minor typo fix. [Ilias Tsitsimpis] #### Changes - --info displays the imaplib2 version and whether it's the bundled or system one. [Nicolas Sebrecht] - Bump from imaplib2 v2.53 to v2.55. [Nicolas Sebrecht] - Move requirements.txt to the root directory. [Nicolas Sebrecht] - README: rename "Requirements" section to "Requirements & dependencies". [Nicolas Sebrecht] - README: add imaplib2 dependency and remove libraries in the standard libraries. [Nicolas Sebrecht] - offlineimap.conf: improved comments. [Nicolas Sebrecht] - sqlite was made mandatory: import error can fail at import time. [Nicolas Sebrecht] - release.sh: put the authors directly to the AUTHORS section. [Nicolas Sebrecht] - release.sh: learn users how to get the requirements file for pip. [Nicolas Sebrecht] - website-doc.sh: include maintenance releases in the list of announces. [Nicolas Sebrecht] - website-doc.sh: announces.yml: fill the page for the links. [Nicolas Sebrecht] - Remove dead code and other code cleanups. [Nicolas Sebrecht] - Style and comments improvements. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0-rc2 (2016-06-04) #### Notes Enable offlineimap to run with Python 3. This feature is still experimental but very welcome those days. Thanks Łukasz Żarnowiecki to work on this! You are all welcome to test offlineimap with Python 3 and report both sucess and failures. Maintainers, we now work with a virtual imaplib2. Under the hood, the imported imaplib2 can be the bundled version or any other (recent enough) imaplib2 provided by the system. If you already package imaplib2 and want to avoid duplication of code, just remove the bundled version of imaplib2 while packaging offlineimap and it should work out of the box. Be care, the filenames have change. #### Authors - Nicolas Sebrecht (9) - Łukasz Żarnowiecki (2) #### Features - Introduce a virtual imaplib2. [Nicolas Sebrecht] - Mark Python 3 supported and experimental. [Nicolas Sebrecht] - Allow to run under python3 without special env. [Łukasz Żarnowiecki] - Maildir: Create top level dir recursively. [Łukasz Żarnowiecki] #### Fixes - IMAP: ignore UID with 0 as value when searching for UIDs. [Nicolas Sebrecht] - Minor: fix copyright date. [Nicolas Sebrecht] #### Changes - Threading: improve comments. [Nicolas Sebrecht] - Bump imaplib2 from v2.52 to v2.53. [Nicolas Sebrecht] - globals: use whitespaces instead of tabs. [Nicolas Sebrecht] - six: add requirements for pip. [Nicolas Sebrecht] - README: add six library requirement. [Nicolas Sebrecht] ### OfflineIMAP v7.0.0-rc1 (2016-05-19) #### Notes We are starting a new major release cycle. The dabatase for the cache is now sqlite by default. This means downgrading to previous versions is prone to errors if you don't have sqlite enabled in your configuration. All users should enable the sqlite database now to avoid issues. Expect the legacy text files to be deprecated and removed in the future. The long time awaited feature to not delete any message while allowing adding new messages and sync flags is now implemented and marked stable. Thanks to the testers and your feedbacks! Łukasz started the work to support Python 3. Because of this, the six dependency is required. If you have scripts using the pid file, be aware this file is no longer used because running multiple instances of the program is supported for years. There a lot of code factorization and documentation improvements, especially around threads. I'm happy new contributors joined the official team, especially Łukasz and Ilias (Debian maintainer). Thank you! #### Authors - Nicolas Sebrecht (32) - Łukasz Żarnowiecki (17) - Dodji Seketeli (1) - Om Prakash (1) #### Features - Make sqlite status cache the default. [Nicolas Sebrecht] - Learn to not delete messages. [Nicolas Sebrecht] - Inform when maxage/startdate is in the future. [Łukasz Żarnowiecki] - offlineimap.conf: XOAUTH2: expose and document the oauth2_request_url option. [Nicolas Sebrecht] - offlineimap.conf: improve documentation for oauth2. [Nicolas Sebrecht] #### Fixes - sqlite: open database when we use it rather than at instantiation time. [Nicolas Sebrecht] - SQLite: close db when done. [Nicolas Sebrecht] - conf: newmail_hook is a remote option. [Nicolas Sebrecht] - folder: utime_from_header is for Maildir only. [Nicolas Sebrecht] - Handle maxage for davmail correctly. [Łukasz Żarnowiecki] - XOAUTH2: don't force oauth2_request_url to be defined. [Nicolas Sebrecht] - XOAUTH2: raise error when oauth_request_url is missing for IMAP type. [Nicolas Sebrecht] - IMAP: don't try to create empty folders. [Nicolas Sebrecht] - Really execute the recipe of the 'docs' target in top-most Makefile. [Dodji Seketeli] #### Changes - release.sh: make no differences between contributors. [Nicolas Sebrecht] - threading: fix variable names about namespaces. [Nicolas Sebrecht] - imapserver: use boolean where it makes sense. [Nicolas Sebrecht] - threading: suggeststhreads must honor CLI and conf options. [Nicolas Sebrecht] - threading: improve variable names and factorize code. [Nicolas Sebrecht] - py3: raise exceptions using six module. [Łukasz Żarnowiecki] - threading: minor improvements. [Nicolas Sebrecht] - instancelimitedsems does not need a lock but must be used with global. [Nicolas Sebrecht] - threading: get rid of the syncaccount function. [Nicolas Sebrecht] - get rid of offlineimap/syncmaster.py. [Nicolas Sebrecht] - threading: rename threadslist to accountThreads. [Nicolas Sebrecht] - threading: simplify names. [Nicolas Sebrecht] - Encode utf-8 argument for md5 function. [Łukasz Żarnowiecki] - Replace dictionary iteration methods. [Łukasz Żarnowiecki] - threading: simplify the monitoring code for threads. [Nicolas Sebrecht] - threadutil: don't limit the number of threads. [Nicolas Sebrecht] - threading: add comments. [Nicolas Sebrecht] - Wrap zip calls with list call. [Łukasz Żarnowiecki] - Remove xreadlines calls. [Łukasz Żarnowiecki] - Replace xrange with range. [Łukasz Żarnowiecki] - Replace has_key method to "key in dict". [Łukasz Żarnowiecki] - Change filter with lambda to list comprehension. [Łukasz Żarnowiecki] - Replace calls to long with int calls. [Łukasz Żarnowiecki] - Add workaround for string.split for Python3. [Łukasz Żarnowiecki] - Convert basestring to str. [Łukasz Żarnowiecki] - Rename email.Parser to email.parser. [Łukasz Żarnowiecki] - Do not mix tabs with spaces. [Łukasz Żarnowiecki] - Convert except X,T to except X as T. [Łukasz Żarnowiecki] - Add tags to gitignore. [Łukasz Żarnowiecki] - don't write a pid file. [Nicolas Sebrecht] - manual: improve rendering. [Nicolas Sebrecht] - manual: improve sqlite section. [Nicolas Sebrecht] - minor: logs: print readonly message in all debug modes. [Nicolas Sebrecht] - accounts.py: minor improvements. [Nicolas Sebrecht] - folder: properly factorize initialization and dropping of self.message. [Nicolas Sebrecht] - offlineimap.txt: minor typo fixes. [Om Prakash] ### OfflineIMAP v6.7.0 (2016-03-10) #### Notes New stable release out! With the work of Ilias, maintainer at Debian, OfflineIMAP is learning a new CLI option to help fixing filenames for the users using nametrans and updating from versions prior to v6.3.5. Distribution maintainers might want to backport this feature for their packaged versions out after v6.3.5. Have a look at commit c84d23b65670f to know more. OfflineIMAP earns the slogan "Get the emails where you need them", authored by Norbert Preining. Julien Danjou, the author of the book _The Hacker’s Guide To Python_, shared us his screenshot of a running session of OfflineIMAP. I recently created rooms for chat sessions at Gitter. It appears to be really cool, supports seamless authentication with a github account, persistent logs, desktop/mobile clients and many more usefull features. Join us at Gitter! - https://gitter.im/OfflineIMAP/offlineimap [NEW] - https://gitter.im/OfflineIMAP/imapfw [NEW] Now, the OfflineIMAP community has 2 official websites: - http://www.offlineimap.org (for offlineimap) - http://imapfw.offlineimap.org (for imapfw) [NEW] The Twitter account was resurrected, too. Feel free to join us: https://twitter.com/OfflineIMAP Finally, the teams of the OfflineIMAP organization at Github were renewed to facilitate the integration of new contributors and directly improve both the documentation and the websites. As a side note, the [imapfw repository](https://github.com/OfflineIMAP/imapfw) has now more than 50 stargazers. This is very encouraging. Thank you much everybody for your various contributions into OfflineIMAP! #### Authors - Ben Boeckel (1) - Ebben Aries (1) - Ilias Tsitsimpis (1) #### Features - Introduce a code of conduct. - Add github templates. - Change hard coding of AF_UNSPEC to user-defined address-families per repository. [Ebben Aries] - Add documentation for the ipv6 configuration option. #### Fixes - Identify and fix messages with FMD5 inconsistencies. [Ilias Tsitsimpis] - Curses, UIBase: remove references to __bigversion__. [Ben Boeckel] - Sphinx doc: remove usage of __bigversion__. - MANIFEST: exclude rfcs (used for Pypi packages). - Changelog: fix typo. #### Changes - release.sh: move the authors section up. - release.sh: add pypi instructions. - MAINTAINERS: update. ### OfflineIMAP v6.7.0-rc2 (2016-02-22) #### Notes Learn to abruptly abort on multiple Ctrl+C. Some bugs got fixed. XOAUTH2 now honors the proxy configuration option. Error message was improved when it fails to write a new mail in a local Maildir. I've enabled the hook for integration with Github. You'll get notifications on updates of the master branch of the repository (mostly for new releases). I may write some tweets about OfflineIMAP sometimes. #### Features - Abort after three Ctrl-C keystrokes. #### Fixes - Fix year of copyright. - Versioning: avoid confusing pip by spliting out __version__ with __revision__. - Fix: exceptions.OSError might not have attribute EEXIST defined. - XOAUTH2 handler: urlopen with proxied socket. - Manual: small grammar fix. - Fix typos in offlineimap(1) manpage. #### Changes - Update links to the new URL www.offlineimap.org. ### OfflineIMAP v6.7.0-rc1 (2016-01-24) #### Notes Starting a new cycle with all EXPERIMENTAL and TESTING stuff marked stable. Otherwise, not much exciting yet. There's pending work that would need some love by contributors: - https://github.com/OfflineIMAP/offlineimap/issues/211 - https://github.com/OfflineIMAP/offlineimap/pull/111 - https://github.com/OfflineIMAP/offlineimap/issues/184 #### Features - Allow authorization via XOAUTH2 using access token. #### Fixes - Revert "Don't output initial blurb in "quiet" mode". - Fix Changelog. #### Changes - Declare newmail_hook option stable. - Declare utime_from_header option stable. - Decode foldernames is removed EXPERIMENTAL flag. - Declare XOAUTH2 stable. - Declare tls_level option stable. - Declare IMAP Keywords option stable. ### OfflineIMAP v6.6.1 (2015-12-28) #### Notes This is a very small new stable release for two fixes. Amending support for BINARY APPEND which is not correctly implemented. Also, remove potential harms from dot files in a local maildir. #### Fixes - Bump imaplib2 from 2.53 to 2.52. Remove support for binary send. - Ignore aloo dot files in the Maildir while scanning for mails. ### OfflineIMAP v6.6.0 (2015-12-05) #### Features - Maildir learns to mimic Dovecot's format of lower-case letters (a,b,c..) for "custom flags" or user keywords. #### Fixes - Broken retry loop would break connection management. - Replace rogue `print` statement by `self.ui.debug`. #### Changes - Bump imaplib2 from v2.52 to v2.53. - Code cleanups. - Add a full stack of all thread dump upon EXIT or KILL signal in thread debug mode. ### OfflineIMAP v6.6.0-rc3 (2015-11-05) #### Notes Changes are slowing down and the code is under serious testing by some new contributors. Everything expected at this time in the release cycle. Thanks to them. SSL is now enabled by default to prevent from sending private data in clear stream to the wild. #### Features - Add new config option `filename_use_mail_timestamp`. #### Fixes - Bump from imaplib2 v2.51 to v2.52. - Minor fixes. #### Changes - Enable SSL by default. - Fix: avoid writing password to log. - offlineimap.conf: improve namtrans doc a bit. ### OfflineIMAP v6.6.0-rc2 (2015-10-15) #### Notes Interesting job was done in this release with 3 new features: - Support for XOAUTH2; - New 'tls_level' configuration option to automatically discard insecure SSL protocols; - New interface 'syslog' comes in, next to the -s CLI option. This allows better integration with systemd. I won't merge big changes until the stable is out. IOW, you can seriously start testing this rc2. #### Features - Add a new syslog ui. - Introduce the 'tls_level' configuration option. - Learn XOAUTH2 authentication (used by Gmail servers). - Manual IDLE section improved (minor). #### Fixes - Configuration option utime_from_header handles out-of-bounds dates. - offlineimap.conf: fix erroneous assumption about ssl23. - Fix status code to reflect success or failure of a sync. - contrib/release.sh: fix changelog edition. #### Changes - Bump imaplib2 from v2.48 to v2.51. - README: new section status and future. - Minor code cleanups. - Makefile: improve building of targz. - systemd: log to syslog rather than stderr for better integration. ### OfflineIMAP v6.6.0-rc1 (2015-09-28) #### Notes Let's go with a new release. Basic UTF support was implemented while it is still exeprimental. Use this with care. OfflineIMAP can now send the logs to syslog and notify on new mail. #### Features - logging: add a switch to log to syslog. - Added the newmail_hook. - utf-7 feature is set experimental. #### Fixes - offlineimap.conf: fix a typo in the new mail hook example. - Fix language. - Fix spelling inconsistency. - offlineimap.conf: don't use quotes for sep option. - man page: fingerprint can be used with SSL. - fix #225 « Runonce (offlineimap -o) does not stop if autorefresh is declared in DEFAULT section ». - CONTRIBUTING: fix links to offlineimap.org. #### Changes - Bump imaplib2 from 2.43 to 2.48 - README: small improvements ### OfflineIMAP v6.5.7 (2015-05-15) #### Notes Almost no change since last release candidate. This is a sign that this release is stable. ,-) There was big changes since previous stable and users - especially distribution maintainers - should really read the intermediate changelogs. At the beginning of this year, I've tried to implement Unicode support. As you know, I was not satisfied with the result. Then, I've published my code analysis where I talk about doing a lot of refactoring for more proper OOP practices. What's new is that I've actually done it and stopped this work as soon as I realized that it means entirely rewriting the software. On top of this, I'm not fully satisfied with other current limitations: - old legacy support; - migration to Python 3; - complex multithreading design; - some restrictions of the GPLv2 license; - etc. That's why I've started a new product. I'll publish it in the coming weeks under the MIT license. #### Features - Better documentation for Windows users. - contrib/release.sh (v0.2): fixes and improvements. #### Fixes - Report exceptions via exit code. - Proxy feature leaks DNS support: offlineimap.conf talks about this. - Email parsing for date coudn't work: fix datetuple dst check. #### Changes - Little code refactoring. ### OfflineIMAP v6.5.7-rc4 (2015-04-07) #### Notes Contrary to what the detailed following changes look like, here is a much bigger release than expected. Most important change is about maxage being sightly revisited. The whole internal logic was found broken. Janna Martl did the hard work of raising the issues and get them fixed. New configuration options are added. Maintainer Dmitrijs Ledkovs has left the organization. We wish you well! ,-) Sebastian Spaeth let us know he will be almost inactive. We wish you well, too! #### Features - Add configuration option "utime_from_header" (TESTING). - Add systemd integration files. - mbnames: add new option "incremental" to write the file once per account. #### Fixes - maxage: fix timezone issues, remove IMAP-IMAP support, add startdate option. - Test suites fixed and improved. - Fix inaccurate UI messages when some messages are internally excluded from the cached lists. #### Changes - imaplib2: bump to v2.43. - More documentations moves to the website. - Maintainer Dmitrijs has left the organization. - Remove unnecessary imaplib2 workaround. - release.sh: script for maintainers improved. ### OfflineIMAP v6.5.7-rc3 (2015-03-19) #### Notes Here comes a much bigger release than expected! With this release, the new website is made official. Distribution maintainers, be aware that we now have a new man page offlineimapui(7)! Also, the man page offlineimap(1) is sightly revised to explain the command line options. Since `offlineimap --help` won't detail the options anymore, it becomes critical. The maxage feature was broken by design and could delete mails on one side. It is still under heavy work to fix issues when timezones are not synced. Gmail is known to use different timezones accross mailboxes. The IMAP library imaplib2 was updated for the upcoming course to Python 3. The most other important changes are: - Possibility to use a proxy. - All the documentation are SIGHTLY revisited and updated from all the available places (sources files in the repository, wiki, website). A lot was moved from the wiki and the sources to the website. - the RFCs are available in the repository. #### Features - Add proxy support powered by PySocks. - New man page offlineimapui to explain the available UIs. - Add a CONTRIBUTING.rst file. - Add a `TODO.rst` list for the contributors. - Add a script for maintainers to roll out new releases. - Add the `scripts/get-repository.sh` script to work on the website and the wiki. - Doc: add IMAP RFCs. #### Fixes - Don't loose local mails because of maxage. - Properly handle the cached messagelist. - Do not error if `remoteuser` is not configured. - imaplibutil: add missing errno import. - LocalStatusSQLite: labels: don't fail if database returns unexpected None value. - IDLE: continue trying selecting the folder on `OfflineImapError.Error`. #### Changes - imaplib2: bump to v2.42 - `--help` becomes concise. - Changelogs: move format back to markdown/kramdown to be more compatible with Jekyll. - README: deep cleanups. - code cleanups. - code: more style consistency. - sqlite: provide offending filename when open fails. - MANUAL: full refactoring, change format to asciidoc. - MANUAL: rename "KNOWN BUGS" TO "KNOWN ISSUES". - MANUAL: add known issues entry about socktimeout for suspended sessions. - offlineimap.conf: say what is the default value for the sep option. - sqlite: provide information on what is failing for `OperationalError`. - remove obsolete documentation. ### OfflineIMAP v6.5.7-rc2 (2015-01-18) #### Notes This release candidate should be minor for most users. The best points are about SSL not falling back on other authentication methods when failing, better RAM footprint and reduced I/O access. Documentation had our attention, too. There's some code cleanups and code refactoring, as usual. #### Features * Do not keep reloading pyhtonfile, make it stateful. * HACKING: how to create tags. * MANUAL: add minor sample on how to retrieve a password with a helper python file. #### Fixes * Make OS-default CA certificate file to be requested explicitely. * SSL: do not fallback on other authentication mode if it fails. * Fix regression introduced while style patching. * API documentation: properly auto-document main class, fixes. * ui: Machine: remove offending param for a _printData() call. * Drop caches after having processed folders. #### Changes * Fix unexpected garbage code. * Properly re-raise exception to save original tracebacks. * Refactoring: avoid redefining various Python keywords. * Code: improvements of comments and more style consistency. * Configuration file: better design and other small improvements. * nametrans documentation: fix minor error. * Unused import removal. * Add a note about the incorrect rendering of the docstring with Sphinx. * Errors handling: log the messages with level ERROR. * MAINTAINERS: add mailing list maintainers. * Fixed copyright statement. * COPYING: fix unexpected characters. ### OfflineIMAP v6.5.7-rc1 (2015-01-07) #### Notes I think it's time for a new release candidate. Our release cycles are long enough and users are asked to use the current TIP of the next branch to test our recent patches. The current version makes better support for environment variable expansion and improves OS portability. Gmail should be better supported: we are still expecting feedbacks. Embedded library imaplib2 is updated to v2.37. Debugging messages are added and polished. There's some code cleanups and refactoring, also. #### Features * Expand environment variables in the following configuration items: - general.pythonfile; - general.metadata; - mbnames.filename; - Repository.localfolders. - Repository.sslcacertfile. Make tilde and environment variable expansion in the following configuration items: - Repository.sslclientcert; - Repository.sslclientkey. * Support default CA bundle locations for a couple of known Unix systems (Michael Vogt, GutHub pull #19) * Added default CA bundle location for OpenBSD (GitHub pull #120) and DragonFlyBSD. #### Fixes * Fix unbounded recursion during flag update (Josh Berry). * Do not ignore gmail labels if header appears multiple times * Delete gmail labels header before adding a new one * Fix improper header separator for X-OfflineIMAP header * Match header names case-insensitively * Create SQLite database directory if it doesn't exist yet; warn if path is not a directory (Nick Farrell, GutHub pull #102) * Properly manipulate contents of messagelist for folder * Fix label processing in GmailMaildir * Properly capitalize OpenSSL * Fix warning-level message processing by MachineUI (GitHub pull #64, GitHub pull #118). * Properly generate tarball from "sdist" command (GitHub #137) * Fix Markdown formatting * Fix typo in apply_xforms invocation * Merge pull request #136 from aroig/gh/label-fix * Fix mangled message headers for servers without UIDPLUS: X-OfflineIMAP was added with preceeding '\n' instead of '\r\n' just before message was uploaded to the IMAP server. * Add missing version bump for 6.5.6 (it was released with 6.5.5 in setup.py and other places). #### Changes * Warn about a tricky piece of code in addmessageheader * Rename addmessageheader()'s crlf parameter to linebreak * addmessageheader: fix case #2 and flesh out docstring * addmessageheader(): add debug for header insertion * Add version qualifier to differentiate releases and development ones * More clearly show results of folder name translation * IMAP: provide message-id in error messages * Trade recursion by plain old cycle * Avoid copying array every time, just slice it * Added OpenSSL exception clause to our main GPL to allow people to link with OpenSSL in run-time. It is needed at least for Debian, see https://lists.debian.org/debian-legal/2002/10/msg00113.html for details. * Brought CustomConfig.py into more proper shape * Updated bundled imaplib2 to 2.37: - add missing idle_lock in _handler() * Imaplib2: trade backticks to repr() * Introduce CustomConfig method that applies set of transforms * imaplibutil.py: remove unused imports * CustomConfig.py: remove unused imports * init.py: remove unused import * repository/Base.py: remove unused import * repository/GmailMaildir.py: remove unused import * repository/LocalStatus.py: remove unused import * ui/Curses.py: remove unused import * ui/UIBase.py: remove unused import * localeval: comment on security issues * docs: remove obsolete comment about SubmittingPatches.rst * utils/const.py: fix ident * ui/UIBase: folderlist(): avoid built-in list() redefinition * more consistent style ### OfflineIMAP v6.5.6 (2014-05-14) * Fix IDLE mode regression (it didn't worked) introduced after v6.5.5 (pointy hat goes to Eygene Ryabinkin, kudos -- to Tomasz Żok) ### OfflineIMAP v6.5.6-rc1 (2014-05-14) * Add knob to invoke folderfilter dynamically on each sync (GitHub#73) * Add knob to apply compression to IMAP connections (Abdó Roig-Maranges) * Add knob to filter some headers before uploading message to IMAP server (Abdó Roig-Maranges) * Allow to sync GMail labels and implement GmailMaildir repository that adds mechanics to change message labels (Abdó Roig-Maranges) * Allow to migrate status data across differend backends (Abdó Roig-Maranges) * Support XDG Base Directory Specification (if $XDG_CONFIG_HOME/offlineimap/config exists, use it as the default configuration path; ~/.offlineimaprc is still tried after XDG location) (GitHub#32) * Allow multiple certificate fingerprints to be specified inside 'cert_fingerprint' ### OfflineIMAP v6.5.5 (2013-10-07) * Avoid lockups for IMAP synchronizations running with the "-1" command-line switch (X-Ryl669 ) * Dump stacktrace for all threads on SIGQUIT: ease debugging of threading and other issues * SIGHUP is now handled as the termination notification rather than the signal to reread the configuration (Dmitrijs Ledkovs) * Honor the timezone of emails (Tobias Thierer) * Allow mbnames output to be sorted by a custom sort key by specifying a 'sort_keyfunc' function in the [mbnames] section of the config. * Support SASL PLAIN authentication method. (Andreas Mack) * Support transport-only tunnels that requre full IMAP authentication. (Steve Purcell) * Make the list of authentication mechanisms to be configurable. (Andreas Mack) * Allow to set message access and modification timestamps based on the "Date" header of the message itself. (Cyril Russo) * "peritem" format string for [mbnames] got new expansion key "localfolders" that corresponds to the same parameter of the local repository for the account being processed. * [regression] pass folder names to the foldersort function, revert the documented behaviour * Fix handling of zero-sized IMAP data items (GitHub#15). * Updated bundled imaplib2 to 2.35: - fix for Gmail sending a BYE response after reading >100 messages in a session; - includes fix for GitHub#15: patch was accepted upstream. * Updated bundled imaplib2 to 2.36: it includes support for SSL version override that was integrated into our code before, no other changes. * Fixed parsing of quoted strings in IMAP responses: strings like "\\" were treated as having \" as the escaped quote, rather than treating it as the quoted escaped backslash (GitHub#53). * Execute pre/post-sync hooks during synchronizations toggled by IMAP IDLE message processing. (maxgerer@gmail.com) * Catch unsuccessful local mail uploads when IMAP server responds with "NO" status; that resulted in a loss of such local messages. (Adam Spiers) * Don't create folders if readonly is enabled. * Learn to deal with readonly folders to properly detect this condition and act accordingly. One example is Gmail's "Chats" folder that is read-only, but contains logs of the quick chats. (E. Ryabinkin) * Fix str.format() calls for Python 2.6 (D. Logie) * Remove APPENDUID hack, previously introduced to fix Gmail, no longer necessary, it might have been breaking things. (J. Wiegley) * Improve regex that could lead to 'NoneType' object has no attribute 'group' (D. Franke) * Improved error throwing on repository misconfiguration ### OfflineIMAP v6.5.4 (2012-06-02) * bump bundled imaplib2 library 2.29 --> 2.33 * Actually perform the SSL fingerprint check (reported by J. Cook) * Curses UI, don't use colors after we shut down curses already (C.Höger) * Document that '%' needs encoding as '%%' in configuration files. * Fix crash when IMAP.quickchanged() led to an Error (reported by sharat87) * Implement the createfolders setting to disable folder propagation (see docs) ### OfflineIMAP v6.5.3.1 (2012-04-03) * Don't fail if no dry-run setting exists in offlineimap.conf (introduced in 6.5.3) ### OfflineIMAP v6.5.3 (2012-04-02) * --dry-run mode protects us from performing any actual action. It will not precisely give the exact information what will happen. If e.g. it would need to create a folder, it merely outputs "Would create folder X", but not how many and which mails it would transfer. * internal code changes to prepare for Python3 * Improve user documentation of nametrans/folderfilter * Fixed some cases where invalid nametrans rules were not caught and we would not propagate local folders to the remote repository. (now tested in test03) * Revert "* Slight performance enhancement uploading mails to an IMAP server in the common case." It might have led to instabilities. * Revamped documentation structure. `make` in the `docs` dir or `make doc` in the root dir will now create the 1) man page and 2) the user documentation using sphinx (requiring python-doctools, and sphinx). The resulting user docs are in `docs/html`. You can also only create the man pages with `make man` in the `docs` dir. * -f command line option only works on the untranslated remote repository folder names now. Previously folderfilters had to match both the local AND remote name which caused unwanted behavior in combination with nametrans rules. Clarify in the help text. * Some better output when using nonsensical configuration settings * Improve compatability of the curses UI with python 2.6 ### OfflineIMAP v6.5.2.1 (2012-04-04) * Fix python2.6 compatibility with the TTYUI backend (crash) * Fix TTYUI regression from 6.5.2 in refresh loop (crash) * Fix crashes related to UIDVALIDITY returning "None" * Beginning of a test suite. So far there is only one test. Configure test/credentials.conf and invoke with "python setup.py test" * Make folders containing quotes work rather than crashing (reported by Mark Eichin) * Improve delete msg performance with SQLITE backend * Enforce basic UI when using the --info switch * Remove the Gmail "realdelete" option, as it could lead to potential data loss. ### OfflineIMAP v6.5.2 (2012-01-17) * Gmail "realdelete" option is considered harmful and has the potential for data loss. Analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 Warnings were added to offlineimap.conf * Rather than write out the nametrans'lated folder names for mbnames, we now write out the local untransformed box names. This is generally what we want. This became relevant since we support nametrans rules on the local side since only a short time. Reported by Paul Collignan. * Some sanity checks and improved error messages. * Revert 6.5.1.1 change to use public imaplib2 function, it was reported to not always work. * Don't fail when ~/netrc is not readable by us. * Don't emit noisy regular sleeping announcements in Basic UI. ### OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" Smallish bug fixes that deserve to be put out. * Fix possible crash during --info run * Fix reading in Maildirs, where we would attempt to create empty directories on REMOTE. * Do not attempt to sync lower case custom Maildir flags. We do not support them (yet) (this prevents many scary bogus sync messages) * Add filter information to the filter list in --info output ### OfflineIMAP v6.5.1.1 (2012-01-07) - "Das machine control is nicht fur gerfinger-poken und mittengrabben" Blinkenlights UI 6.5.0 regression fixes only. * Sleep led to crash ('abort_signal' not existing) * Make exit via 'q' key work again cleanly ### OfflineIMAP v6.5.1 (2012-01-07) - "Quest for stability" * Fixed Maildir regression "flagmatchre" not found. (regressed in 6.5.0) * Have console output go by default to STDOUT and not STDERR (regression in 6.5.0) * Fixed MachineUI to urlencode() output lines again, rather than outputting multi-line items. It's ugly as hell, but it had been that way for years. * Remove the old global locking system. We lock only the accounts that we currently sync, so you can invoke OfflineImap multiple times now as long as you sync different accounts. This system is compatible with all releases >= 6.4.0, so don't run older releases simultanous to this one. ### OfflineIMAP v6.5.0 (2012-01-06) This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know that the 6.4.x series is problematic. * Uploading multiple emails to an IMAP server would lead to wrong UIDs being returned (ie the same for all), which confused offlineimap and led to recurrent upload/download loops and inconsistencies in the IMAP<->IMAP uid mapping. * Uploading of Messages from Maildir and IMAP<->IMAP has been made more efficient by renaming files/mapping entries, rather than actually loading and saving the message under a new UID. * Fix regression that broke MachineUI ### OfflineIMAP v6.4.4 (2012-01-06) This is a bugfix release, fixing regressions occurring in or since 6.4.0. * Fix the missing folder error that occured when a new remote folder was detected (IMAP<->Maildir) * Possibly fixed bug that prevented us from ever re-reading Maildir folders, so flag changes and deletions were not detected when running in a refresh loop. This is a regression that was introduced in about 6.4.0. * Never mangle maildir file names when using nonstandard Maildir flags (such as 'a'), note that they will still be deleted as they are not supported in the sync to an IMAP server. ### OfflineIMAP v6.4.3 (2012-01-04) #### New Features * add a --info command line switch that outputs useful information about the server and the configuration for all enabled accounts. #### Changes * Reworked logging which was reported to e.g. not flush output to files often enough. User-visible changes: a) console output goes to stderr (for now). b) file output has timestamps and looks identical in the basic and ttyui UIs. c) File output should be flushed after logging by default (do report if not). * Bumped bundled imaplib2 to release 2.29 * Make ctrl-c exit cleanly rather aborting brutally (which could leave around temporary files, half-written cache files, etc). Exiting on SIGTERM and CTRL-C can take a little longer, but will be clean. ### OfflineIMAP v6.4.2 (2011-12-01) * IMAP<->IMAP sync with a readonly local IMAP repository failed with a rather mysterious "TypeError: expected a character buffer object" error. Fix this my retrieving the list of folders early enough even for readonly repositories. * Fix regression from 6.4.0. When using local Maildirs with "/" as a folder separator, all folder names would get a trailing slash appended, which is plain wrong. ### OfflineIMAP v6.4.1 (2011-11-17) #### Changes * Indicate progress when copying many messages (slightly change log format) * Output how long an account sync took (min:sec). #### Bug Fixes * Syncing multiple accounts in single-threaded mode would fail as we try to "register" a thread as belonging to two accounts which was fatal. Make it non-fatal (it can be legitimate). * New folders on the remote would be skipped on the very sync run they are created and only by synced in subsequent runs. Fixed. * a readonly parameter to select() was not always treated correctly, which could result in some folders being opened read-only when we really needed read-write. ### OfflineIMAP v6.4.0 (2011-09-29) This is the first stable release to support the forward-compatible per-account locks and remote folder creation that has been introduced in the 6.3.5 series. * Various regression and bug fixes from the last couple of RCs ### OfflineIMAP v6.3.5-rc3 (2011-09-21) #### Changes * Refresh server capabilities after login, so we know that Gmail supports UIDPLUS (it only announces that after login, not before). This prevents us from adding custom headers to Gmail uploads. #### Bug Fixes * Fix the creation of folders on remote repositories, which was still botched on rc2. ### OfflineIMAP v6.3.5-rc2 (2011-09-19) #### New Features * Implement per-account locking, so that it will possible to sync different accounts at the same time. The old global lock is still in place for backward compatibility reasons (to be able to run old and new versions of OfflineImap concurrently) and will be removed in the future. Starting with this version, OfflineImap will be forward-compatible with the per-account locking style. * Implement RFC 2595 LOGINDISABLED. Warn the user and abort when we attempt a plaintext login but the server has explicitly disabled plaintext logins rather than crashing. * Folders will now also be automatically created on the REMOTE side of an account if they exist on the local side. Use the folderfilters setting on the local side to prevent some folders from migrating to the remote side. Also, if you have a nametrans setting on the remote repository, you might need a nametrans setting on the local repository that leads to the original name (reverse nametrans). #### Changes * Documentation improvements concerning 'restoreatime' and some code cleanup * Maildir repositories now also respond to folderfilter= configurations. #### Bug Fixes * New emails are not created with "-rwxr-xr-x" but as "-rw-r--r--" anymore, fixing a regression in 6.3.4. ### OfflineIMAP v6.3.5-rc1 (2011-09-12) #### Notes Idle feature and SQLite backend leave the experimental stage! ,-) #### New Features * When a message upload/download fails, we do not abort the whole folder synchronization, but only skip that message, informing the user at the end of the sync run. * If you connect via ssl and 'cert_fingerprint' is configured, we check that the server certificate is actually known and identical by comparing the stored sha1 fingerprint with the current one. #### Changes * Refactor our IMAPServer class. Background work without user-visible changes. * Remove the configurability of the Blinkenlights statuschar. It cluttered the main configuration file for little gain. * Updated bundled imaplib2 to version 2.28. #### Bug Fixes * We protect more robustly against asking for inexistent messages from the IMAP server, when someone else deletes or moves messages while we sync. * Selecting inexistent folders specified in folderincludes now throws nice errors and continues to sync with all other folders rather than exiting offlineimap with a traceback. ### OfflineIMAP v6.3.4 (2011-08-10) #### Notes Here we are. A nice release since v6.3.3, I think. #### Changes * Handle when UID can't be found on saved messages. ### OfflineIMAP v6.3.4-rc4 (2011-07-27) #### Notes There is nothing exciting in this release. This is somewhat expected due to the late merge on -rc3. #### New Features * Support maildir for Windows. #### Changes * Manual improved. ### OfflineIMAP v6.3.4-rc3 (2011-07-07) #### Notes Here is a surprising release. :-) As expected we have a lot bug fixes in this round (see git log for details), including a fix for a bug we had for ages (details below) which is a very good news. What makes this cycle so unusual is that I merged a feature to support StartTLS automatically (thanks Sebastian!). Another very good news. We usually don't do much changes so late in a cycle. Now, things are highly calming down and I hope a lot of people will test this release. Next one could be the stable! #### New Features * Added StartTLS support, it will automatically be used if the server supports it. #### Bug Fixes * We protect more robustly against asking for inexistent messages from the IMAP server, when someone else deletes or moves messages while we sync. ### OfflineIMAP v6.3.4-rc2 (2011-06-15) #### Notes This was a very active rc1 and we could expect a lot of new fixes for the next release. The most important fix is about a bug that could lead to data loss. Find more information about his bug here: http://permalink.gmane.org/gmane.mail.imap.offlineimap.general/3803 The IDLE support is merged as experimental feature. #### New Features * Implement experimental IDLE feature. #### Changes * Maildirs use less memory while syncing. #### Bug Fixes * Saving to Maildirs now checks for file existence without race conditions. * A bug in the underlying imap library has been fixed that could potentially lead to data loss if the server interrupted responses with unexpected but legal server status responses. This would mainly occur in folders with many thousands of emails. Upgrading from the previous release is strongly recommended. ### OfflineIMAP v6.3.4-rc1 (2011-05-16) #### Notes Welcome to the v6.3.4 pre-release cycle. Your favorite IMAP tool wins 2 new features which were asked for a long time: * an experimental SQL-based backend for the local cache; * one-way synchronization cabability. Logic synchronization is reviewed and simplified (from 4 to 3 passes) giving improved performance. Lot of work was done to give OfflineIMAP a better code base. Raised errors can now rely on a new error system and should become the default in the coming releases. As usual, we ask our users to test this release as much as possible, especially the SQL backend. Have fun! #### New Features * Begin sphinx-based documentation for the code. * Enable 1-way synchronization by settting a [Repository ...] to readonly = True. When e.g. using offlineimap for backup purposes you can thus make sure that no changes in your backup trickle back into the main IMAP server. * Optional: experimental SQLite-based backend for the LocalStatus cache. Plain text remains the default. #### Changes * Start a enhanced error handling background system. This is designed to not stop a whole sync process on all errors (not much used, yet). * Documentation improvements: the FAQ wins new entries and add a new HACKING file for developers. * Lot of code cleanups. * Reduced our sync logic from 4 passes to 3 passes (integrating upload of "new" and "existing" messages into one function). This should result in a slight speedup. * No whitespace is stripped from comma-separated arguments passed via the -f option. * Give more detailed error when encountering a corrupt UID mapping file. #### Bug Fixes * Drop connection if synchronization failed. This is needed if resuming the system from suspend mode gives a wrong connection. * Fix the offlineimap crash when invoking debug option 'thread'. * Make 'thread' command line option work. ### OfflineIMAP v6.3.3 (2011-04-24) #### Notes Make this last candidate cycle short. It looks like we don't need more tests as most issues were raised and solved in the second round. Also, we have huge work to merge big and expected features into OfflineIMAP. Thanks to all contributors, again. With such a contribution rate, we can release stable faster. I hope it will be confirmed in the longer run! #### Changes * Improved documentation for querying password. ### OfflineIMAP v6.3.3-rc3 (2011-04-19) #### Notes It's more than a week since the previous release. Most of the issues raised were discussed and fixed since last release. I think we can be glad and confident for the future while the project live his merry life. #### Changes * The -f option did not work with Folder names with spaces. It works now, use with quoting e.g. -f "INBOX, Deleted Mails". * Improved documentation. * Bump from imaplib2 v2.20 to v2.22. * Code refactoring. #### Bug Fixes * Fix IMAP4 tunnel with imaplib2. ### OfflineIMAP v6.3.3-rc2 (2011-04-07) #### Notes We are now at the third week of the -rc1 cycle. I think it's welcome to begin the -rc2 cycle. Things are highly calming down in the code even if we had much more feedbacks than usual. Keep going your effort! I'd like to thank reporters who involved in this cycle: - Баталов Григорий - Alexander Skwar - Christoph Höger - dtk - Greg Grossmeier - h2oz7v - Iain Dalton - Pan Tsu - Vincent Beffara - Will Styler (my apologies if I forget somebody) ...and all active developers, of course! The imaplib2 migration looks to go the right way to be definetly released but still needs more tests. So, here we go... #### Changes * Increase compatability with Gmail servers which claim to not support the UIDPLUS extension but in reality do. #### Bug Fixes * Fix hang when using Ctrl+C in some cases. ### OfflineIMAP v6.3.3-rc1 (2011-03-16) #### Notes Here is time to begin the tests cycle. If feature topics are sent, I may merge or delay them until the next stable release. Main change comes from the migration from imaplib to imaplib2. It's internal code changes and doesn't impact users. UIDPLUS and subjectAltName for SSL are also great improvements. This release includes a hang fix due to infinite loop. Users seeing OfflineIMAP hang and consuming a lot of CPU are asked to update. That beeing said, this is still an early release candidate you should use for non-critical data only! #### New Features * Implement UIDPLUS extension support. OfflineIMAP will now not insert an X-OfflineIMAP header if the mail server supports the UIDPLUS extension. * SSL: support subjectAltName. #### Changes * Use imaplib2 instead of imaplib. * Makefile use magic to find the version number. * Rework the repository module * Change UI names to Blinkenlights,TTYUI,Basic,Quiet,MachineUI. Old names will still work, but are deprecated. Document that we don't accept a list of UIs anymore. * Reworked the syncing strategy. The only user-visible change is that blowing away LocalStatus will not require you to redownload ALL of your mails if you still have the local Maildir. It will simply recreate LocalStatus. * TTYUI ouput improved. * Code cleanups. #### Bug Fixes * Fix ignoring output while determining the rst2xxx command name to build documentation. * Fix hang because of infinite loop reading EOF. * Allow SSL connections to send keep-alive messages. * Fix regression (UIBase is no more). * Make profiling mode really enforce single-threading * Do not send localized date strings to the IMAP server as it will either ignore or refuse them. ### OfflineIMAP v6.3.2 (2010-02-21) #### Notes First of all I'm really happy to announce our new official `website `_. Most of the work started from the impulse of Philippe LeCavalier with the help of Sebastian Spaeth and other contributors. Thanks to everybody. In this release, we are still touched by the "SSL3 write pending" but I think time was long enough to try to fix it. We have our first entry in the "KNOWN BUG" section of the manual about that. I'm afraid it could impact a lot of users if some distribution package any SSL library not having underlying (still obscure) requirements. Distribution maintainers should be care of it. I hope this release will help us to have more reports. This release will also be the root of our long maintenance support. Other bugs were fixed. #### Bug Fixes * Fix craches for getglobalui(). * Fix documentation build. * Restore compatibiliy with python 2.5. ### OfflineIMAP v6.3.2-rc3 (2010-02-06) #### Notes We are still touched by the "SSL3 write pending" bug it would be really nice to fix before releasing the coming stable. In the worse case, we'll have to add the first entry in the "KNOWN BUG" section of the manual. I'm afraid it could impact a lot of users if some distribution package any SSL library not having underlying (still obscure) requirements. The best news with this release are the Curse UI fixed and the better reports on errors. In this release I won't merge any patch not fixing a bug or a security issue. More feedbacks on the main issue would be appreciated. #### Changes * Sample offlineimap.conf states it expects a PEM formatted certificat. * Give better trace information if an error occurs. * Have --version ONLY print the version number. * Code cleanups. #### Bug Fixes * Fix Curses UI (simplified by moving from MultiLock to Rlock implementation). * Makefile: docutils build work whether python extension command is stripped or not. * Makefile: clean now removes HTML documentation files. ### OfflineIMAP v6.3.2-rc2 (2010-12-21) #### Notes We are beginning a new tests cycle. At this stage, I expect most people will try to intensively stuck OfflineIMAP. :-) #### New Features * Makefile learn to build the package and make it the default. * Introduce a Changelog to involve community in the releasing process. * Migrate documentation to restructuredtext. #### Changes * Improve CustomConfig documentation. * Imply single threading mode in debug mode exept for "-d thread". * Code and import cleanups. * Allow UI to have arbitrary names. * Code refactoring around UI and UIBase. * Improve version managment and make it easier. * Introduce a true single threading mode. #### Bug Fixes * Understand multiple EXISTS replies from servers like Zimbra. * Only verify hostname if we actually use CA cert. * Fix ssl ca-cert in the sample configuration file. * Fix 'Ctrl+C' interruptions in threads. * Fix makefile clean for files having whitespaces. * Fix makefile to not remove unrelated files. * Fixes in README. * Remove uneeded files. ### OfflineIMAP v6.3.2-rc1 (2010-12-19) #### Notes We are beginning a tests cycle. If feature topics are sent, I may merge or delay them until the next stable release. #### New Features * Primitive implementation of SSL certificates check. #### Changes * Use OptionParser instead of getopts. * Code cleanups. #### Bug Fixes * Fix reading password from UI. ### OfflineIMAP v6.3.1 (2010-12-11) #### Notes Yes, I know I've just annouced the v6.3.0 in the same week. As said, it was not really a true release for the software. This last release includes fixes and improvements it might be nice to update to. Thanks to every body who helped to make this release with patches and tips through the mailing list. This is clearly a release they own. #### Changes * cProfile becomes the default profiler. Sebastian Spaeth did refactoring to prepare to the coming unit test suites. * UI output formating enhanced. * Some code cleanups. #### Bug Fixes * Fix possible overflow while working with Exchange. * Fix time sleep while exiting threads. ### OfflineIMAP v6.3.0 (2010-12-09) #### Notes This release is more "administrative" than anything else and mainly marks the change of the maintainer. New workflow and policy for developers come in. BTW, I don't think I'll maintain debian/changelog. At least, not in the debian way. Most users and maintainers may rather want to skip this release. #### Bug Fixes * Fix terminal display on exit. * netrc password authentication. * User name querying from netrc. ================================================ FILE: MAINTAINERS.rst ================================================ .. -*- coding: utf-8 -*- Contacts ======== - Abdó Roig-Maranges - email: abdo.roig at gmail.com - github: aroig - Ben Boeckel - email: mathstuf at gmail.com - github: mathstuf - benutzer193 - email: registerbn at gmail.com - github: benutzer193 - Chris Coleman - email: chris at espacenetworks.com - github: chris001 - Darshit Shah - email: darnir at gmail.com - github: darnir - Eygene Ryabinkin - email: rea at freebsd.org - github: konvpalto - other: FreeBSD maintainer - Igor Almeida - email: igor.contato at gmail.com - github: igoralmeida - Ilias Tsitsimpis - email: i.tsitsimpis at gmail.com - github: iliastsi - other: Debian maintainer - "J" - email: offlineimap at 927589452.de - github: 927589452 - other: FreeBSD user - Łukasz Żarnowiecki - email: dolohow at outlook.com - github: dolohow - Nicolas Sebrecht - email: nicolas.s-dev at laposte.net - github: nicolas33 - system: Linux - Remi Locherer - email: remi.locherer at relo.ch - system: OpenBSD maintainer - Sebastian Spaeth - email: sebastian at sspaeth.de - github: spaetz - other: left the project but still responding Testers ======= - Abdó Roig-Maranges - Ben Boeckel - Chris Coleman - Darshit Shah - Eygene Ryabinkin - Igor Almeida - Ilias Tsitsimpis - "J" - Łukasz Żarnowiecki - Nicolas Sebrecht - Remi Locherer Maintainers =========== - Eygene Ryabinkin - Sebastian Spaeth - Nicolas Sebrecht - Chris Coleman Github ------ - Eygene Ryabinkin - Sebastian Spaeth - Nicolas Sebrecht Mailing List ------------ - Eygene Ryabinkin - Sebastian Spaeth - Nicolas Sebrecht Twitter ------- - Nicolas Sebrecht Pypi ---- - Nicolas Sebrecht - Sebastian Spaeth ================================================ FILE: MANIFEST.in ================================================ global-exclude .gitignore .git *.bak *.orig *.rej include setup.py include COPYING include Changelog* include MAINTAINERS include MANIFEST.in include Makefile include README.md include offlineimap.conf* include offlineimap.py recursive-include contrib * recursive-include offlineimap *.py recursive-include bin * recursive-include docs * recursive-include test * prune docs/rfcs ================================================ FILE: Makefile ================================================ # Copyright (C) 2002 - 2018 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Warning: VERSION, ABBREV and TARGZ are used in docs/build-uploads.sh. VERSION=$(shell ./offlineimap.py --version) ABBREV=$(shell git log --format='%h' HEAD~1..) TARGZ=offlineimap-v$(VERSION)-$(ABBREV) SHELL=/bin/bash RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` all: build build: python setup.py build @echo @echo "Build process finished, run 'python setup.py install' to install" \ "or 'python setup.py --help' for more information". clean: -python setup.py clean --all -rm -f bin/offlineimapc 2>/dev/null -find . -name '*.pyc' -exec rm -f {} \; -find . -name '*.pygc' -exec rm -f {} \; -find . -name '*.class' -exec rm -f {} \; -find . -name '.cache*' -exec rm -f {} \; -find . -type d -name '__pycache__' -exec rm -rf {} \; -rm -f manpage.links manpage.refs 2>/dev/null -find . -name auth -exec rm -vf {}/password {}/username \; -$(MAKE) -C docs clean .PHONY: docs docs: @$(MAKE) -C docs websitedoc: @$(MAKE) -C websitedoc targz: ../$(TARGZ) ../$(TARGZ): cd .. && tar -zhcv --transform s,^offlineimap,offlineimap-v$(VERSION), -f $(TARGZ).tar.gz --exclude '.*.swp' --exclude '.*.swo' --exclude '*.pyc' --exclude '__pycache__' offlineimap/{bin,Changelog.md,Changelog.maint.md,contrib,CONTRIBUTING.rst,COPYING,docs,MAINTAINERS.rst,Makefile,MANIFEST.in,offlineimap,offlineimap.conf,offlineimap.conf.minimal,offlineimap.py,README.md,requirements.txt,scripts,setup.cfg,setup.py,snapcraft.yaml,test,tests,TODO.rst} rpm: targz cd .. && sudo rpmbuild -ta $(TARGZ) ================================================ FILE: README.md ================================================ Upstream status (`master` branch): [![OfflineIMAP build status on Travis-CI.org](https://travis-ci.org/OfflineIMAP/offlineimap.svg?branch=master)](https://travis-ci.org/OfflineIMAP/offlineimap) [![OfflineIMAP code coverage on Codecov.io](https://codecov.io/gh/OfflineIMAP/offlineimap/branch/master/graph/badge.svg)](https://codecov.io/gh/OfflineIMAP/offlineimap) [![Gitter chat](https://badges.gitter.im/OfflineIMAP/offlineimap.png)](https://gitter.im/OfflineIMAP/offlineimap) Upstream status (`next` branch): [![OfflineIMAP build status on Travis-CI.org](https://travis-ci.org/OfflineIMAP/offlineimap.svg?branch=next)](https://travis-ci.org/OfflineIMAP/offlineimap) [offlineimap]: http://github.com/OfflineIMAP/offlineimap [website]: http://www.offlineimap.org [wiki]: http://github.com/OfflineIMAP/offlineimap/wiki [blog]: http://www.offlineimap.org/posts.html Links: * Official github code repository: [offlineimap] * Website: [website] * Wiki: [wiki] * Blog: [blog] # OfflineIMAP ***"Get the emails where you need them."*** [Official offlineimap][offlineimap]. ## Description OfflineIMAP is software that downloads your email mailbox(es) as **local Maildirs**. OfflineIMAP will synchronize both sides via *IMAP*. ## Why should I use OfflineIMAP? IMAP's main downside is that you have to **trust** your email provider to not lose your email. While certainly unlikely, it's not impossible. With OfflineIMAP, you can download your Mailboxes and make you own backups of your [Maildir](https://en.wikipedia.org/wiki/Maildir). This allows reading your email offline without the need for your mail reader (MUA) to support IMAP operations. Need an attachment from a message without internet connection? No problem, the message is still there. ## Project status and future > As one of the maintainer of OfflineIMAP, I'd like to put my efforts into > [imapfw](http://github.com/OfflineIMAP/imapfw). **imapfw** is software in > development that I intend to replace OfflineIMAP with in the long term. > > That's why I'm not going to continue OfflineIMAP development. I'll continue > to maintain OfflineIMAP (fixing small bugs, reviewing patches and merging, > and rolling out new releases), but that's all. > > While I keep tracking issues for OfflineIMAP, you should not expect future support. > > You won't be left at the side. OfflineIMAP's community is large enough so that > you'll find people for most of your issues. > > Get news from the [blog][blog]. > > Nicolas Sebrecht. ,-) ## License GNU General Public License v2. ## Downloads You should first check if your distribution already packages OfflineIMAP for you. Downloads releases as [tarball or zipball](https://github.com/OfflineIMAP/offlineimap/tags). If you are running Linux Os, you can install offlineimap with: - openSUSE `zypper in offlineimap` - archLinux `pacman -S offlineimap` - fedora `dnf install offlineimap` ## Feedbacks and contributions **The user discussions, development, announcements and all the exciting stuff take place on the mailing list.** While not mandatory to send emails, you can [subscribe here](http://lists.alioth.debian.org/mailman/listinfo/offlineimap-project). Bugs, issues and contributions can be requested to both the mailing list or the [official Github project][offlineimap]. Provide the following information: - system/distribution (with version) - offlineimap version (`offlineimap -V`) - Python version - server name or domain - CLI options - Configuration file (offlineimaprc) - pythonfile (if any) - Logs, error - Steps to reproduce the error ## The community * OfflineIMAP's main site is the [project page at Github][offlineimap]. * There is the [OfflineIMAP community's website][website]. * And finally, [the wiki][wiki]. ## Requirements & dependencies * Python v2.7+ * six (required) * rfc6555 (required) * imaplib2 >= 2.57 (optional) * gssapi (optional), for Kerberos authentication * portalocker (optional), if you need to run offlineimap in Cygwin for Windows * Python v3.4+ ***[STALLED] (experimental: [see known issues](https://github.com/OfflineIMAP/offlineimap/issues?q=is%3Aissue+is%3Aopen+label%3APy3))*** ## Documentation All current and updated documentation is on the [community's website][website]. ### Read documentation locally You might want to read the documentation locally. Get the sources of the website. For the other documentation, run the appropriate make target: ```sh $ ./scripts/get-repository.sh website $ cd docs $ make html # Requires rst2html $ make man # Requires a2x (http://asciidoc.org) $ make api # Requires sphinx ``` ================================================ FILE: TODO.rst ================================================ .. vim: spelllang=en ts=2 expandtab : .. _coding style: https://github.com/OfflineIMAP/offlineimap/blob/next/docs/CodingGuidelines.rst ============================ TODO list by relevance order ============================ Should be the starting point to improve the `coding style`_. Write your WIP directly in this file. TODO list --------- * Better names for variables, objects, etc. * Improve comments. Most of the current comments assume a very good knowledge of the internals. That sucks because I guess nobody is anymore aware of ALL of them. Time when this was a one guy made project has long passed. * Better policy on objects. - Turn ALL attributes private and use accessors. This is not "pythonic" but such pythonic thing turn the code into intricated code. - Turn ALL methods not intended to be used outside, private. * Revamp the factorization. It's not unusual to find "factorized" code for bad reasons: because it made the code /look/ nicer, but the factorized function/methods is actually called from ONE place. While it might locally help, such practice globally defeat the purpose because we lose the view of what is true factorized code and what is not. * Namespace the factorized code. If a method require a local function, DON'T USE yet another method. Use a local namespaced function.:: class BLah(object): def _internal_method(self, arg): def local_factorized(local_arg): # local_factorized's code # _internal_method's code. Python allows local namespaced functions for good reasons. * Better inheritance policy. Take the sample of the folder/LocalStatus(SQlite) and folder/Base stuffs. It's *nearly IMPOSSIBLE* to know and understand what parent method is used by what child, for what purpose, etc. So, instead of (re)defining methods in the wild, keep the well common NON-redefined stuff into the parent and define the required methods in the childs. We really don't want anything like:: def method(self): raise NotImplemented While this is common practice in Python, think about that again: how a parent object should know all the expected methods/accessors of all the possible kind of childs? Inheritance is about factorizing, certainly **NOT** about **defining the interface** of the childs. * Introduce as many as intermediate inherited objects as required. Keeping linear inheritance is good because Python sucks at playing with multiple parents and it keeps things simple. But a parent should have ALL its methods used in ALL the childs. If not, it's a good sign that a new intermediate object should be introduced in the inheritance line. * Don't blindly inherit from library objects. We do want **well defined interfaces**. For example, we do too much things like imapobj.methodcall() while the imapobj is far inherited from imaplib2. We have NO clue about what we currently use from the library. Having a dump wrappper for each call should be made mandatory for objects inherited from a library. Using composed objects should be seriously considered in this case, instead of using inheritance. * Use factories. Current objects do too much initialization stuff varying with the context it is used. Move things like that into factories and keep the objects definitions clean. * Make it clear when we expect a composite object and what we expect exactly. Even the more obvious composed objects are badly defined. For example, the ``conf`` instances are spread across a lot of objects. Did you know that such composed objects are sometimes restricted to the section the object works on, and most of the time it's not restricted at all? How many time it requires to find and understand on what we are currently working? * Seriously improve our debugging/hacking sessions (AGAIN). Until now, we have limited the improvements to allow better/full stack traces. While this was actually required, we now hit some limitations of the whole exception-based paradigm. For example, it's very HARD to follow an instance during its life time. I have a good overview of what we could do in this area, so don't matter much about that if you don't get the point or what could be done. * Support Unicode. ================================================ FILE: bin/offlineimap ================================================ #!/usr/bin/env python2 # Startup from system-wide installation # Copyright (C) 2002-2018 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import OfflineImap oi = OfflineImap() oi.run() ================================================ FILE: contrib/README.md ================================================ README ====== **This "./contrib" directory is where users share their own scripts and tools.** Everything here is submitted and maintained *by the users for the users*. You're welcome to add your own stuff. There is no barrier on your contributions here. We think it's expected to find contributions of various quality. ================================================ FILE: contrib/helpers.py ================================================ """ Put into Public Domain, by Nicolas Sebrecht. Helpers for maintenance scripts. """ from os import chdir, makedirs, system, getcwd from os.path import expanduser import shlex from subprocess import check_output, check_call, CalledProcessError import yaml FS_ENCODING = 'UTF-8' ENCODING = 'UTF-8' MAILING_LIST = 'offlineimap-project@lists.alioth.debian.org' CACHEDIR = '.git/offlineimap-release' EDITOR = 'vim' MAILALIASES_FILE = expanduser('~/.mutt/mail_aliases') TESTERS_FILE = "{}/testers.yml".format(CACHEDIR) ME = "Nicolas Sebrecht " def run(cmd): return check_output(cmd, timeout=5).rstrip() def goTo(path): try: chdir(path) return True except FileNotFoundError: print("Could not find the '{}' directory in '{}'...".format( path, getcwd()) ) return False class Author(object): def __init__(self, name, count, email): self.name = name self.count = count self.email = email def getName(self): return self.name def getCount(self): return self.count def getEmail(self): return self.email class Git(object): @staticmethod def getShortlog(ref): shortlog = "" cmd = shlex.split("git shortlog --no-merges -n v{}..".format(ref)) output = run(cmd).decode(ENCODING) for line in output.split("\n"): if len(line) > 0: if line[0] != " ": line = " {}\n".format(line) else: line = " {}\n".format(line.lstrip()) else: line = "\n" shortlog += line return shortlog @staticmethod def add(files): cmd = shlex.split("git add -- {}".format(files)) return run(cmd).decode(ENCODING) @staticmethod def commit(msg): cmd = shlex.split("git commit -s -m '{}'".format(msg)) return run(cmd).decode(ENCODING) @staticmethod def tag(version): cmd = shlex.split("git tag -a 'v{}' -m 'v{}'".format(version, version)) return run(cmd).decode(ENCODING) @staticmethod def stash(msg): cmd = shlex.split("git stash create '{}'".format(msg)) return run(cmd).decode(ENCODING) @staticmethod def mergeFF(ref): cmd = shlex.split("git merge --ff '{}'".format(ref)) return run(cmd).decode(ENCODING) @staticmethod def getDiffstat(ref): cmd = shlex.split("git diff --stat v{}..".format(ref)) return run(cmd).decode(ENCODING) @staticmethod def isClean(): try: check_call(shlex.split("git diff --quiet")) check_call(shlex.split("git diff --cached --quiet")) except CalledProcessError: return False return True @staticmethod def buildMessageId(): cmd = shlex.split( "git log HEAD~1.. --oneline --pretty='%H.%t.upcoming.%ce'") return run(cmd).decode(ENCODING) @staticmethod def resetKeep(ref): return run(shlex.split("git reset --keep {}".format(ref))) @staticmethod def getRef(ref): return run(shlex.split("git rev-parse {}".format(ref))).rstrip() @staticmethod def rmTag(tag): return run(shlex.split("git tag -d {}".format(tag))) @staticmethod def checkout(ref, create=False): if create: create = "-b" else: create = "" cmd = shlex.split("git checkout {} {}".format(create, ref)) run(cmd) head = shlex.split("git rev-parse HEAD") revparseRef = shlex.split("git rev-parse {}".format(ref)) if run(head) != run(revparseRef): raise Exception("checkout to '{}' did not work".format(ref)) @staticmethod def makeCacheDir(): try: makedirs(CACHEDIR) except FileExistsError: pass @staticmethod def getLocalUser(): cmd = shlex.split("git config --get user.name") name = run(cmd).decode(ENCODING) cmd = shlex.split("git config --get user.email") email = run(cmd).decode(ENCODING) return name, email @staticmethod def buildDate(): cmd = shlex.split("git log HEAD~1.. --oneline --pretty='%cD'") return run(cmd).decode(ENCODING) @staticmethod def getAuthorsList(sinceRef): authors = [] cmd = shlex.split("git shortlog --no-merges -sne v{}..".format(sinceRef)) output = run(cmd).decode(ENCODING) for line in output.split("\n"): count, full = line.strip().split("\t") full = full.split(' ') name = ' '.join(full[:-1]) email = full[-1] authors.append(Author(name, count, email)) return authors @staticmethod def getCommitsList(sinceRef): cmd = shlex.split( "git log --no-merges --format='- %h %s. [%aN]' v{}..".format(sinceRef) ) return run(cmd).decode(ENCODING) @staticmethod def chdirToRepositoryTopLevel(): cmd = shlex.split("git rev-parse --show-toplevel") topLevel = run(cmd) chdir(topLevel) class OfflineimapInfo(object): def getVersion(self): cmd = shlex.split("./offlineimap.py --version") return run(cmd).rstrip().decode(FS_ENCODING) def editInit(self): return system("{} ./offlineimap/__init__.py".format(EDITOR)) class User(object): """Interact with the user.""" @staticmethod def request(msg, prompt='--> '): print(msg) return input(prompt) @staticmethod def pause(msg=False): return User.request(msg, prompt="Press Enter to continue..") @staticmethod def yesNo(msg, defaultToYes=False, prompt='--> '): endMsg = " [y/N]: No" if defaultToYes: endMsg = " [Y/n]: Yes" msg += endMsg answer = User.request(msg, prompt).lower() if answer in ['y', 'yes']: return True if defaultToYes is not False and answer not in ['n', 'no']: return True return False class Tester(object): def __init__(self, name, email, feedback): self.name = name self.email = email self.feedback = feedback def __str__(self): return "{} {}".format(self.name, self.email) def getName(self): return self.name def getEmail(self): return self.email def getFeedback(self): return self.feedback def positiveFeedback(self): return self.feedback is True def setFeedback(self, feedback): assert feedback in [True, False, None] self.feedback = feedback def switchFeedback(self): self.feedback = not self.feedback class Testers(object): def __init__(self): self.testers = None self._read() def _read(self): self.testers = [] with open(TESTERS_FILE, 'r') as fd: testers = yaml.load(fd) for tester in testers: name = tester['name'] email = tester['email'] feedback = tester['feedback'] self.testers.append(Tester(name, email, feedback)) self.testers.sort(key=lambda x: x.getName().lower()) @staticmethod def listTestersInTeam(): """Returns a list of emails extracted from my mailaliases file.""" cmd = shlex.split("grep offlineimap-testers {}".format(MAILALIASES_FILE)) output = run(cmd).decode(ENCODING) emails = output.lstrip("alias offlineimap-testers ").split(', ') return emails def add(self, name, email, feedback=None): self.testers.append(Tester(name, email, feedback)) def remove(self, tester): self.testers.remove(tester) def get(self): return self.testers def getList(self): testersList = "" for tester in self.testers: testersList += "- {}\n".format(tester.getName()) return testersList def getListOk(self): testersOk = [] for tester in self.testers: if tester.positiveFeedback(): testersOk.append(tester) return testersOk def reset(self): for tester in self.testers: tester.setFeedback(None) def write(self): testers = [] for tester in self.testers: testers.append({ 'name': tester.getName(), 'email': tester.getEmail(), 'feedback': tester.getFeedback(), }) with open(TESTERS_FILE, 'w') as fd: fd.write(yaml.dump(testers)) ================================================ FILE: contrib/internet-urllib3.py ================================================ #!/usr/bin/env python import urllib3 import certifi def isInternetConnected(url="www.ietf.org"): result = False http = urllib3.PoolManager( cert_reqs='CERT_REQUIRED', # Force certificate check. ca_certs=certifi.where(), # Path to the Certifi bundle. ) try: r = http.request('HEAD', 'https://' + url) result = True except Exception as e: # urllib3.exceptions.SSLError result = False return result print isInternetConnected() ================================================ FILE: contrib/release.py ================================================ #!/usr/bin/python3 """ Put into Public Domain, by Nicolas Sebrecht. Make a new release. """ #TODO: announce: cc list on announce includes all testers #TODO: announce: remove empty sections #TODO: websitedoc up #TODO: website branch not including all changes! from os import system, path, rename from datetime import datetime from subprocess import check_call import shlex import time from email import utils from helpers import ( MAILING_LIST, CACHEDIR, EDITOR, Git, OfflineimapInfo, Testers, User, run, goTo ) __VERSION__ = "0.2" SPHINXBUILD = 'sphinx-build' DOCSDIR = 'docs' CHANGELOG_MAGIC = '{:toc}' WEBSITE_LATEST = "website/_data/latest.yml" CHANGELOG_EXCERPT = "{}/changelog.excerpt.md".format(CACHEDIR) CHANGELOG_EXCERPT_OLD = "{}.old".format(CHANGELOG_EXCERPT) CHANGELOG = "Changelog.md" ANNOUNCE_FILE = "{}/announce.txt".format(CACHEDIR) WEBSITE_LATEST_SKEL = """# DO NOT EDIT MANUALLY: it is generated by the release script. stable: v{stable} """ CHANGELOG_SKEL = """ ### OfflineIMAP v{version} ({date}) #### Notes This release was tested by: {testersList} #### Authors {authorsList} #### Features #### Fixes #### Changes {commitsList} """ END_MESSAGE = """ Release is ready! Make your checks and push the changes for both offlineimap and the website. Announce template stands in '{announce}'. Command samples to do manually: - git push master next {new_version} - python setup.py sdist && twine upload dist/* && rm -rf dist MANIFEST - cd website - git checkout master - git merge {website_branch} - git push master - cd .. - git send-email {announce} ...and write a Twitter message. Have fun! ,-) """ class State(object): def __init__(self): self.master = None self.next = None self.website = None self.tag = None def setTag(self, tag): self.tag = tag def save(self): self.master = Git.getRef('master') self.next = Git.getRef('next') def saveWebsite(self): Git.chdirToRepositoryTopLevel() goTo('website') self.website = Git.getRef('master') goTo('..') def restore(self): Git.chdirToRepositoryTopLevel() try: Git.checkout('-f') except: pass # Git.checkout('master') # Git.resetKeep(self.master) # Git.checkout('next') # Git.resetKeep(self.next) if self.tag is not None: Git.rmTag(self.tag) if self.website is not None: if goTo('website'): Git.checkout(self.website) goTo('..') class Changelog(object): def __init__(self): self.shouldUsePrevious = False def edit(self): return system("{} {}".format(EDITOR, CHANGELOG_EXCERPT)) def update(self): # Insert excerpt to CHANGELOG. system("sed -i -e '/{}/ r {}' '{}'".format( CHANGELOG_MAGIC, CHANGELOG_EXCERPT, CHANGELOG ) ) # Remove trailing whitespaces. system("sed -i -r -e 's, +$,,' '{}'".format(CHANGELOG)) def savePrevious(self): rename(CHANGELOG_EXCERPT, CHANGELOG_EXCERPT_OLD) def isPrevious(self): if path.isfile(CHANGELOG_EXCERPT_OLD): return True return False def showPrevious(self): output = run(shlex.split("cat '{}'".format(CHANGELOG_EXCERPT_OLD))) for line in output.splitlines(): print(line.decode('utf-8')) # Weird to have to decode bytes here. def usePrevious(self): rename(CHANGELOG_EXCERPT_OLD, CHANGELOG_EXCERPT) self.shouldUsePrevious = True def usingPrevious(self): return self.shouldUsePrevious def writeExcerpt(self, version, date, testersList, authorsList, commitsList): with open(CHANGELOG_EXCERPT, 'w+') as fd: fd.write(CHANGELOG_SKEL.format( version=version, date=date, testersList=testersList, authorsList=authorsList, commitsList=commitsList, )) def getSectionsContent(self): dict_Content = {} with open(CHANGELOG_EXCERPT, 'r') as fd: currentSection = None for line in fd: line = line.rstrip() if line == "#### Notes": currentSection = 'Notes' dict_Content['Notes'] = "" continue # Don't keep this title. elif line == "#### Authors": currentSection = 'Authors' dict_Content['Authors'] = "" continue # Don't keep this title. elif line == "#### Features": currentSection = 'Features' dict_Content['Features'] = "" continue # Don't keep this title. elif line == "#### Fixes": currentSection = 'Fixes' dict_Content['Fixes'] = "" continue # Don't keep this title. elif line == "#### Changes": currentSection = 'Changes' dict_Content['Changes'] = "" continue # Don't keep this title. elif line == "-- ": break # Stop extraction. if currentSection is not None: dict_Content[currentSection] += "{}\n".format(line) #TODO: cleanup empty sections. return dict_Content class Announce(object): def __init__(self, version): self.fd = open(ANNOUNCE_FILE, 'w') self.version = version def setHeaders(self, messageId, date): self.fd.write("Message-Id: {}\n".format(messageId)) self.fd.write("Date: {}\n".format(date)) self.fd.write("From: Nicolas Sebrecht \n") self.fd.write("To: {}\n".format(MAILING_LIST)) self.fd.write( "Subject: [ANNOUNCE] OfflineIMAP v{} released\n".format(self.version)) self.fd.write("\n") self.fd.write(""" OfflineIMAP v{version} is out. Downloads: http://github.com/OfflineIMAP/offlineimap/archive/v{version}.tar.gz http://github.com/OfflineIMAP/offlineimap/archive/v{version}.zip Pip: wget "https://raw.githubusercontent.com/OfflineIMAP/offlineimap/v{version}/requirements.txt" -O requirements.txt pip install -r ./requirements.txt --user git+https://github.com/OfflineIMAP/offlineimap.git@v{version} """.format(version=self.version) ) def setContent(self, dict_Content): self.fd.write("\n") for section in ['Notes', 'Authors', 'Features', 'Fixes', 'Changes']: if section in dict_Content: if section != "Notes": self.fd.write("# {}\n".format(section)) self.fd.write(dict_Content[section]) self.fd.write("\n") # Signature. self.fd.write("-- \n") self.fd.write("Nicolas Sebrecht\n") def close(self): self.fd.close() class Website(object): def updateUploads(self): req = ("add new archive to uploads/ on the website? " "(warning: checksums will change if it already exists)") if User.yesNo(req, defaultToYes=True) is False: return False if check_call(shlex.split("./docs/build-uploads.sh")) != 0: return exit(5) return True def updateAPI(self): req = "update API of the website? (requires {})".format(SPHINXBUILD) if User.yesNo(req, defaultToYes=True) is False: return False try: if check_call(shlex.split("{} --version".format(SPHINXBUILD))) != 0: raise RuntimeError("{} not found".format(SPHINXBUILD)) except: print(""" Oops! you don't have {} installed?" Cannot update the webite documentation..." You should install it and manually run:" $ cd {}" $ make websitedoc" Then, commit and push changes of the website.""".format(SPHINXBUILD, DOCSDIR)) User.pause() return False Git.chdirToRepositoryTopLevel() if not goTo('website'): User.pause() return False if not Git.isClean: print("There is WIP in the website repository: stashing") Git.stash('WIP during offlineimap API import') goTo('..') return True def buildLatest(self, version): Git.chdirToRepositoryTopLevel() with open(WEBSITE_LATEST, 'w') as fd: fd.write(WEBSITE_LATEST_SKEL.format(stable=version)) def exportDocs(self): if not goTo(DOCSDIR): User.pause() return if check_call(shlex.split("make websitedoc")) != 0: print("error while calling 'make websitedoc'") exit(3) def createImportBranch(self, version): branchName = "import-v{}".format(version) Git.chdirToRepositoryTopLevel() if not goTo("website"): User.pause() return Git.checkout(branchName, create=True) Git.add('.') Git.commit("update for offlineimap v{}".format(version)) User.pause( "website: branch '{}' is ready for a merge in master!".format( branchName ) ) goTo('..') return branchName class Release(object): def __init__(self): self.state = State() self.offlineimapInfo = OfflineimapInfo() self.testers = Testers() self.changelog = Changelog() self.websiteBranch = "NO_BRANCH_NAME_ERROR" def getVersion(self): return self.offlineimapInfo.getVersion() def prepare(self): if not Git.isClean(): print("The git repository is not clean; aborting") exit(1) Git.makeCacheDir() Git.checkout('next') def requestVersion(self, currentVersion): User.request("going to make a new release after {}".format(currentVersion)) def updateVersion(self): self.offlineimapInfo.editInit() def checkVersions(self, current, new): if new == current: print("version was not changed; stopping.") exit(1) def updateChangelog(self): if self.changelog.isPrevious(): self.changelog.showPrevious() if User.yesNo("A previous Changelog excerpt was found. Use it?"): self.changelog.usePrevious() if not self.changelog.usingPrevious(): date = datetime.now().strftime('%Y-%m-%d') testersList = "" testers = self.testers.getListOk() authorsList = "" authors = Git.getAuthorsList(currentVersion) for tester in testers: testersList += "- {}\n".format(tester.getName()) for author in authors: authorsList += "- {} ({})\n".format( author.getName(), author.getCount() ) commitsList = Git.getCommitsList(currentVersion) date = datetime.now().strftime('%Y-%m-%d') self.changelog.writeExcerpt( newVersion, date, testersList, authorsList, commitsList ) self.changelog.edit() self.changelog.update() def writeAnnounce(self): announce = Announce(newVersion) messageId = utils.make_msgid('release.py', 'laposte.net') nowtuple = datetime.now().timetuple() nowtimestamp = time.mktime(nowtuple) date = utils.formatdate(nowtimestamp) announce.setHeaders(messageId, date) announce.setContent(self.changelog.getSectionsContent()) announce.close() def make(self): Git.add('offlineimap/__init__.py') Git.add('Changelog.md') commitMsg = "v{}\n".format(newVersion) for tester in self.testers.getListOk(): commitMsg = "{}\nTested-by: {} {}".format( commitMsg, tester.getName(), tester.getEmail() ) Git.commit(commitMsg) self.state.setTag(newVersion) Git.tag(newVersion) Git.checkout('master') Git.mergeFF('next') Git.checkout('next') def updateWebsite(self, newVersion): self.state.saveWebsite() website = Website() website.buildLatest(newVersion) res_upload = website.updateUploads() res_api = website.updateAPI() if res_api: res_export = website.exportDocs() if True in [res_upload, res_api, res_export]: self.websiteBranch = website.createImportBranch(newVersion) def getWebsiteBranch(self): return self.websiteBranch def after(self): for protectedRun in [self.testers.reset, self.changelog.savePrevious]: try: protectedRun() except Exception as e: print(e) def restore(self): self.state.restore() if __name__ == '__main__': release = Release() Git.chdirToRepositoryTopLevel() try: release.prepare() currentVersion = release.getVersion() release.requestVersion(currentVersion) release.updateVersion() newVersion = release.getVersion() release.checkVersions(currentVersion, newVersion) release.updateChangelog() release.writeAnnounce() User.pause() release.make() release.updateWebsite(newVersion) release.after() websiteBranch = release.getWebsiteBranch() print(END_MESSAGE.format( announce=ANNOUNCE_FILE, new_version=newVersion, website_branch=websiteBranch) ) except Exception as e: release.restore() raise ================================================ FILE: contrib/release.sh ================================================ #!/bin/sh # # Put into Public Domain, by Nicolas Sebrecht # # Create new releases in OfflineIMAP. # TODO: https://developer.github.com/v3/repos/releases/#create-a-release # https://developer.github.com/libraries/ # https://github.com/turnkeylinux/octohub # https://github.com/michaelliao/githubpy (onefile) # https://github.com/sigmavirus24/github3.py # https://github.com/copitux/python-github3 # https://github.com/PyGithub/PyGithub # https://github.com/micha/resty (curl) # TODO: move configuration out and source it. # TODO: implement rollback. __VERSION__='v0.3' SPHINXBUILD=sphinx-build MAILING_LIST='offlineimap-project@lists.alioth.debian.org' GITHUB_FILE_LINK_PREFIX='https://raw.githubusercontent.com/OfflineIMAP/offlineimap' DOCSDIR='docs' ANNOUNCE_MAGIC='#### Notes ' CHANGELOG_MAGIC='{:toc}' CHANGELOG='Changelog.md' CACHEDIR='.git/offlineimap-release' WEBSITE='website' WEBSITE_LATEST="${WEBSITE}/_data/latest.yml" TMP_CHANGELOG_EXCERPT="${CACHEDIR}/changelog.excerpt.md" TMP_CHANGELOG_EXCERPT_OLD="${TMP_CHANGELOG_EXCERPT}.old" TMP_CHANGELOG="${CACHEDIR}/changelog.md" TMP_ANNOUNCE="${CACHEDIR}/announce.txt" True=0 False=1 Yes=$True No=$False DEBUG=$True # # $1: EXIT_CODE # $2..: message function die () { n=$1 shift echo $* exit $n } function debug () { if test $DEBUG -eq $True then echo "DEBUG: $*" >&2 fi } # # $1: question # $2: message on abort # function ask () { echo echo -n "--- $1 " read -r ans test "n$ans" = 'n' -o "n$ans" = 'ny' && return $Yes test "n$ans" = "ns" -o "n$ans" = 'nn' && return $No die 1 "! $2" } # # $1: message # $1: path to file # function edit_file () { ask "Press Enter to $1" test $? -eq $Yes && { $EDITOR "$2" reset } } function fix_pwd () { debug 'in fix_pwd' cd "$(git rev-parse --show-toplevel)" || \ die 2 "cannot determine the root of the repository" } function prepare_env () { debug 'in prepare_env' mkdir "$CACHEDIR" 2>/dev/null test ! -d "$CACHEDIR" && die 5 "Could not make cache directory $CACHEDIR" } function check_dirty () { debug 'in check_dirty' git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { die 4 "Commit all your changes first!" } } function welcome () { debug 'in welcome' cat <' : yes, continue - 'n' : no - 's' : skip (ONLY where applicable, otherwise continue) Any other key will abort the program. EOF ask 'Ready?' } function checkout_next () { debug 'in checkout_next' git checkout --quiet next || { die 6 "Could not checkout 'next' branch" } } function merge_maint () { debug 'in merge_maint' git merge --quiet -Xours maint || { die 7 "Could not merge 'maint' branch" } } function get_version () { debug 'in get_version' echo "v$(./offlineimap.py --version)" } function update_offlineimap_version () { debug 'in update_offlineimap_version' edit_file 'update the version in __init__.py' offlineimap/__init__.py } # # $1: previous version # function get_git_history () { debug 'in get_git_history' git log --format='- %h %s. [%aN]' --no-merges "${1}.." } # # $1: previous version # function get_git_who () { debug 'in get_git_who' git shortlog --no-merges -sn "${1}.." | \ sed -r -e 's, +([0-9]+)\t(.*),- \2 (\1),' } # # $1: new version # $2: shortlog function changelog_template_part1 () { debug 'in changelog_template_part1' cat < "$TMP_CHANGELOG_EXCERPT" get_git_who "$2" >> "$TMP_CHANGELOG_EXCERPT" changelog_template_part2 >> "$TMP_CHANGELOG_EXCERPT" get_git_history "$2" >> "$TMP_CHANGELOG_EXCERPT" edit_file "the Changelog excerpt" $TMP_CHANGELOG_EXCERPT # Remove comments. grep -v '//' "$TMP_CHANGELOG_EXCERPT" > "${TMP_CHANGELOG_EXCERPT}.nocomment" mv -f "${TMP_CHANGELOG_EXCERPT}.nocomment" "$TMP_CHANGELOG_EXCERPT" fi # Write new Changelog. cat "$CHANGELOG" > "$TMP_CHANGELOG" debug "include excerpt $TMP_CHANGELOG_EXCERPT to $TMP_CHANGELOG" sed -i -e "/${CHANGELOG_MAGIC}/ r ${TMP_CHANGELOG_EXCERPT}" "$TMP_CHANGELOG" debug 'remove trailing whitespaces' sed -i -r -e 's, +$,,' "$TMP_CHANGELOG" # Remove trailing whitespaces. debug "copy to $TMP_CHANGELOG -> $CHANGELOG" cp -f "$TMP_CHANGELOG" "$CHANGELOG" # Check and edit Changelog. ask "Next step: you'll be asked to review the diff of $CHANGELOG" while true do git diff -- "$CHANGELOG" | less ask 'edit Changelog?' $CHANGELOG test ! $? -eq $Yes && break # Asked to edit the Changelog; will loop again. $EDITOR "$CHANGELOG" done } # # $1: new version # function git_release () { debug 'in git_release' git commit -as -m"$1" git tag -a "$1" -m"$1" git checkout master git merge next git checkout next } function get_last_rc () { git tag | grep -E '^v([0-9][\.-]){3}rc' | sort -V | tail -n1 } function get_last_stable () { git tag | grep -E '^v([0-9][\.])+' | grep -v '\-rc' | sort -V | tail -n1 } function update_website_releases_info() { cat > "$WEBSITE_LATEST" < /dev/null 2>&1 if test ! $? -eq 0 then echo "Oops! you don't have $SPHINXBUILD installed?" echo "Cannot update the webite documentation..." echo "You should install it and run:" echo " $ cd docs" echo " $ make websitedoc" echo "Then, commit and push changes of the website." ask 'continue' return fi # Check website sources are available. cd website if test ! $? -eq 0 then echo "ERROR: cannot go to the website sources" ask 'continue' return fi # Stash any WIP in the website sources. git diff --quiet 2>/dev/null && git diff --quiet --cached 2>/dev/null || { echo "There is WIP in the website repository, stashing" echo "git stash create 'WIP during offlineimap API import'" git stash create 'WIP during offlineimap API import' ask 'continue' } cd .. # Back to offlineimap.git. update_website_releases_info cd "./$DOCSDIR" # Enter the docs directory in offlineimap.git. # Build the docs! make websitedoc && { # Commit changes in a branch. cd ../website # Enter the website sources. branch_name="import-$1" git checkout -b "$branch_name" git add '_doc/versions' git commit -a -s -m"update for offlineimap $1" echo "website: branch '$branch_name' ready for a merge in master!" } ask 'website updated locally; continue' fi } function git_username () { git config --get user.name } function git_usermail () { git config --get user.email } # # $1: new version # function announce_header () { cat < Date: $(git log HEAD~1.. --oneline --pretty='%cD') From: $(git_username) <$(git_usermail)> To: $MAILING_LIST Subject: [ANNOUNCE] OfflineIMAP $1 released OfflineIMAP $1 is out. Downloads: http://github.com/OfflineIMAP/offlineimap/archive/${1}.tar.gz http://github.com/OfflineIMAP/offlineimap/archive/${1}.zip Pip: wget "${GITHUB_FILE_LINK_PREFIX}/${1}/requirements.txt" -O requirements.txt pip install -r ./requirements.txt --user git+https://github.com/OfflineIMAP/offlineimap.git@${1} EOF } function announce_footer () { cat < "$TMP_ANNOUNCE" grep -v '^### OfflineIMAP' "$TMP_CHANGELOG_EXCERPT" | \ grep -v '^#### Notes' >> "$TMP_ANNOUNCE" sed -i -r -e "s,^$ANNOUNCE_MAGIC,," "$TMP_ANNOUNCE" sed -i -r -e "s,^#### ,# ," "$TMP_ANNOUNCE" announce_footer >> "$TMP_ANNOUNCE" } function edit_announce () { edit_file 'edit announce' "$TMP_ANNOUNCE" } # # run # function run () { debug 'in run' fix_pwd check_dirty prepare_env checkout_next clear welcome if test -f "$TMP_CHANGELOG_EXCERPT" then head "$TMP_CHANGELOG_EXCERPT" ask "A previous Changelog excerpt (head above) was found, use it?" if test ! $? -eq $Yes then mv -f "$TMP_CHANGELOG_EXCERPT" "$TMP_CHANGELOG_EXCERPT_OLD" fi fi previous_version="$(get_version)" message="Safety check: release after version:" ask "$message $previous_version ?" update_offlineimap_version new_version="$(get_version)" ask "Safety check: make a new release with version: '$new_version'" "Clear changes and restart" update_changelog "$new_version" "$previous_version" build_announce "$new_version" "$previous_version" edit_announce # Wait for the mainline and announce to be built to not include commits from # maint. merge_maint git_release $new_version # Wait for all the Changelogs to be up-to-date in next. update_website $new_version } run cat < master next $new_version - python setup.py sdist && twine upload dist/* && rm -rf dist MANIFEST - cd website - git checkout master - git merge $branch_name - git push master - cd .. - git send-email $TMP_ANNOUNCE Have fun! ,-) EOF # vim: expandtab ts=2 : ================================================ FILE: contrib/store-pw-with-gpg/README.md ================================================ # gpg-offlineimap Python bindings for offlineimap to use gpg instead of storing cleartext passwords Author: Lorenzo G. [GitHub](https://github.com/lorenzog/gpg-offlineimap) ## Quickstart Requirements: a working GPG set-up. Ideally with gpg-agent. Should work out of the box on most modern Linux desktop environments. 1. Enable IMAP in gmail (if you have two factor authentication, you need to create an app-specific password) 2. Create a directory `~/Mail` 3. In `~/Mail`, create a password file `passwords-gmail.txt`. Format: `account@gmail.com password`. Look at the example file in this directory. 4. **ENCRYPT** the file: `gpg -e passwords-gmail.txt`. It should create a file `passwords-gmail.txt.gpg`. Check you can decrypt it: `gpg -d passwords-gmail.txt.gpg`: it will ask you for your GPG password and show it to you. 5. Use the file `offlineimaprc.sample` as a sample for your own `.offlineimaprc`; edit it by following the comments. Minimal items to configure: the `remoteuser` field and the `pythonfile` parameter pointing at the `offlineimap.py` file in this directory. 6. Run it: `offlineimap`. It should ask you for your GPG passphrase to decrypt the password file. 7. If all works well, delete the cleartext password file. ================================================ FILE: contrib/store-pw-with-gpg/gpg-pw.py ================================================ #!/usr/bin/python # Originally taken from: http://stevelosh.com/blog/2012/10/the-homely-mutt/ # by Steve Losh # Modified by Lorenzo Grespan on Jan, 2014 import re import subprocess from sys import argv import logging from os.path import expanduser import unittest import os import sys logging.basicConfig(level=logging.INFO) DEFAULT_PASSWORDS_FILE = os.path.join( os.path.expanduser('~/Mail'), 'passwords.gpg') def get_keychain_pass(account=None, server=None): '''Mac OSX keychain password extraction''' params = { 'security': '/usr/bin/security', 'command': 'find-internet-password', 'account': account, 'server': server, 'keychain': expanduser('~') + '/Library/Keychains/login.keychain', } command = ("%(security)s -v %(command)s" " -g -a %(account)s -s %(server)s %(keychain)s" % params) output = subprocess.check_output( command, shell=True, stderr=subprocess.STDOUT) outtext = [l for l in output.splitlines() if l.startswith('password: ')][0] return find_password(outtext) def find_password(text): '''Helper method for osx password extraction''' # a non-capturing group r = re.match(r'password: (?:0x[A-F0-9]+ )?"(.*)"', text) if r: return r.group(1) else: logging.warn("Not found") return None def get_gpg_pass(account, storage): '''GPG method''' command = ("gpg", "-d", storage) # get attention print '\a' # BEL output = subprocess.check_output(command) # p = subprocess.Popen(command, stdout=subprocess.PIPE) # output, err = p.communicate() for line in output.split('\n'): r = re.match(r'{} ([a-zA-Z0-9]+)'.format(account), line) if r: return r.group(1) return None def get_pass(account=None, server=None, passwd_file=None): '''Main method''' if not passwd_file: storage = DEFAULT_PASSWORDS_FILE else: storage = os.path.join( os.path.expanduser('~/Mail'), passwd_file) if os.path.exists('/usr/bin/security'): return get_keychain_pass(account, server) if os.path.exists(storage): logging.info("Using {}".format(storage)) return get_gpg_pass(account, storage) else: logging.warn("No password file found") sys.exit(1) return None # test with: python -m unittest # really basic tests.. nothing to see. move along class Tester(unittest.TestCase): def testMatchSimple(self): text = 'password: "exampleonetimepass "' self.assertTrue(find_password(text)) def testMatchComplex(self): text = r'password: 0x74676D62646D736B646970766C66696B0A "anotherexamplepass\012"' self.assertTrue(find_password(text)) if __name__ == "__main__": print get_pass(argv[1], argv[2], argv[3]) ================================================ FILE: contrib/store-pw-with-gpg/offlineimaprc.sample ================================================ [general] # GPG quirks, leave unconfigured ui = ttyui # you can use any name as long as it matches the 'account1, 'account2' in the rest # of the file accounts = account1, account2 # this is where the `gpg-pw.py` file is on disk pythonfile=~/where/is/the/file/gpg-pw.py fsync = False # you can call this any way you like [Account account1] localrepository = account1-local remoterepository = account1-remote # no need to touch this status_backend = sqlite [Account account2] localrepository = account2-local remoterepository = account2-remote status_backend = sqlite # thi sis a gmail account [Repository account1-local] type = Maildir # create with maildirmake or by hand by creating cur, new, tmp localfolders = ~/Mail/Mailboxes/account1 # standard Gmail stuff nametrans = lambda folder: { 'drafts': '[Gmail]/Drafts', 'sent': '[Gmail]/Sent mail', 'flagged': '[Gmail]/Starred', 'trash': '[Gmail]/Trash', 'archive': '[Gmail]/All Mail' }.get(folder, folder) [Repository account1-remote] maxconnections = 1 type = Gmail ssl=yes # for osx, you might need to download the certs by hand #sslcacertfile=~/Mail/certs.pem #sslcacertfile=~/Mail/imap.gmail.com.pem # sslcacertfile=/etc/ssl/cert.pem # or use Linux's standard certs sslcacertfile=/etc/ssl/certs/ca-certificates.crt # your account remoteuser = account1@gmail.com remotepasseval = get_pass(account="account1@gmail.com", server="imap.gmail.com", passwd_file="passwords-gmail.txt.gpg") realdelete = no createfolders = no nametrans = lambda folder: {'[Gmail]/Drafts': 'drafts', '[Gmail]/Sent Mail': 'sent', '[Gmail]/Starred': 'star', '[Gmail]/Trash': 'trash', '[Gmail]/All Mail': 'archive', }.get(folder, folder) folderfilter = lambda folder: folder not in ['[Gmail]/Trash', '[Gmail]/Spam', ] [Repository account2-remote] # copy the stanza above, change the 'account' parameter of get_pass, etc. ================================================ FILE: contrib/store-pw-with-gpg/passwords-gmail.txt ================================================ account1@gmail.com password1 account2@gmail.com password2 ================================================ FILE: contrib/systemd/README.md ================================================ --- layout: page title: Integrating OfflineIMAP into systemd author: Ben Boeckel date: 2015-03-22 contributors: Abdo Roig-Maranges, benutzer193, Hugo Osvaldo Barrera updated: 2017-06-01 --- ## Systemd units These unit files are meant to be used in the user session. You may drop them into `/etc/systemd/user` or `${XDG_DATA_HOME}/systemd/user` followed by `systemctl --user daemon-reload` to have systemd aware of the unit files. These files are meant to be triggered either manually using `systemctl --user start offlineimap.service` or by enabling the timer unit using `systemctl --user enable offlineimap-oneshot.timer`. Additionally, specific accounts may be triggered by using `offlineimap@myaccount.timer` or `offlineimap-oneshot@myaccount.service`. If the defaults provided by these units doesn't suit your setup, any of the values may be overridden by using `systemctl --user edit offlineimap.service`. This'll prevent having to copy-and-edit the original file. ================================================ FILE: contrib/systemd/offlineimap-oneshot.service ================================================ [Unit] Description=Offlineimap Service (oneshot) Documentation=man:offlineimap(1) [Service] Type=oneshot ExecStart=/usr/bin/offlineimap -o -u basic # Give 120 seconds for offlineimap to gracefully stop before hard killing it: TimeoutStopSec=120 [Install] WantedBy=mail.target ================================================ FILE: contrib/systemd/offlineimap-oneshot.timer ================================================ [Unit] Description=Offlineimap Query Timer [Timer] OnBootSec=1m OnUnitInactiveSec=15m [Install] WantedBy=default.target ================================================ FILE: contrib/systemd/offlineimap-oneshot@.service ================================================ [Unit] Description=Offlineimap Service for account %i (oneshot) Documentation=man:offlineimap(1) [Service] Type=oneshot ExecStart=/usr/bin/offlineimap -o -a %i -u basic # Give 120 seconds for offlineimap to gracefully stop before hard killing it. TimeoutStopSec=120 [Install] WantedBy=default.target ================================================ FILE: contrib/systemd/offlineimap-oneshot@.timer ================================================ [Unit] Description=Offlineimap Query Timer for account %i [Timer] OnBootSec=1m OnUnitInactiveSec=15m [Install] WantedBy=default.target ================================================ FILE: contrib/systemd/offlineimap.service ================================================ [Unit] Description=Offlineimap Service Documentation=man:offlineimap(1) [Service] ExecStart=/usr/bin/offlineimap -u basic Restart=on-failure RestartSec=60 [Install] WantedBy=default.target ================================================ FILE: contrib/systemd/offlineimap@.service ================================================ [Unit] Description=Offlineimap Service for account %i Documentation=man:offlineimap(1) [Service] ExecStart=/usr/bin/offlineimap -a %i -u basic Restart=on-failure RestartSec=60 [Install] WantedBy=default.target ================================================ FILE: contrib/tested-by.py ================================================ #!/usr/bin/python3 """ Put into Public Domain, by Nicolas Sebrecht. Manage the feedbacks of the testers for the release notes. """ from os import system import argparse from helpers import CACHEDIR, EDITOR, Testers, User, Git class App(object): def __init__(self): self.args = None self.testers = Testers() self.feedbacks = None def _getTestersByFeedback(self): if self.feedbacks is not None: return self.feedbacks feedbackOk = [] feedbackNo = [] for tester in self.testers.get(): if tester.getFeedback() is True: feedbackOk.append(tester) else: feedbackNo.append(tester) for array in [feedbackOk, feedbackNo]: array.sort(key=lambda t: t.getName()) self.feedbacks = feedbackOk + feedbackNo def parseArgs(self): parser = argparse.ArgumentParser(description='Manage the feedbacks.') parser.add_argument('--add', '-a', dest='add_tester', help='Add tester') parser.add_argument('--delete', '-d', dest='delete_tester', type=int, help='Delete tester NUMBER') parser.add_argument('--list', '-l', dest='list_all_testers', action='store_true', help='List the testers') parser.add_argument('--switchFeedback', '-s', dest='switch_feedback', action='store_true', help='Switch the feedback of a tester') self.args = parser.parse_args() def run(self): if self.args.list_all_testers is True: self.listTesters() if self.args.switch_feedback is True: self.switchFeedback() elif self.args.add_tester: self.addTester(self.args.add_tester) elif type(self.args.delete_tester) == int: self.deleteTester(self.args.delete_tester) def addTester(self, strTester): try: splitted = strTester.split('<') name = splitted[0].strip() email = "<{}".format(splitted[1]).strip() except Exception as e: print(e) print("expected format is: 'Firstname Lastname '") exit(2) self.testers.add(name, email) self.testers.write() def deleteTester(self, number): self.listTesters() removed = self.feedbacks.pop(number) self.testers.remove(removed) print("New list:") self.feedbacks = None self.listTesters() print("Removed: {}".format(removed)) ans = User.request("Save on disk? (s/Q)").lower() if ans in ['s']: self.testers.write() def listTesters(self): self._getTestersByFeedback() count = 0 for tester in self.feedbacks: feedback = "ok" if tester.getFeedback() is not True: feedback = "no" print("{:02d} - {} {}: {}".format( count, tester.getName(), tester.getEmail(), feedback ) ) count += 1 def switchFeedback(self): self._getTestersByFeedback() msg = "Switch tester: [/s/q]" self.listTesters() number = User.request(msg) while number.lower() not in ['s', 'save', 'q', 'quit']: if number == '': continue try: number = int(number) self.feedbacks[number].switchFeedback() except (ValueError, IndexError) as e: print(e) exit(1) finally: self.listTesters() number = User.request(msg) if number in ['s', 'save']: self.testers.write() self.listTesters() def reset(self): self.testers.reset() self.testers.write() #def updateMailaliases(self): if __name__ == '__main__': Git.chdirToRepositoryTopLevel() app = App() app.parseArgs() app.run() ================================================ FILE: contrib/upcoming.py ================================================ #!/usr/bin/python3 """ Put into Public Domain, by Nicolas Sebrecht. Produce the "upcoming release" notes. """ from os import system from helpers import ( MAILING_LIST, CACHEDIR, EDITOR, Testers, Git, OfflineimapInfo, User ) UPCOMING_FILE = "{}/upcoming.txt".format(CACHEDIR) UPCOMING_HEADER = "{}/upcoming-header.txt".format(CACHEDIR) # Header is like: # #Message-Id: <{messageId}> #Date: {date} #From: {name} <{email}> #To: {mailinglist} #Cc: {ccList} #Subject: [ANNOUNCE] upcoming offlineimap v{expectedVersion} # ## Notes # #I think it's time for a new release. # #I aim to make the new release in one week, approximately. If you'd like more #time, please let me know. ,-) # #Please, send me a mail to confirm it works for you. This will be written in the #release notes and the git logs. # # ## Authors # if __name__ == '__main__': offlineimapInfo = OfflineimapInfo() print("Will read headers from {}".format(UPCOMING_HEADER)) Git.chdirToRepositoryTopLevel() oVersion = offlineimapInfo.getVersion() ccList = Testers.listTestersInTeam() authors = Git.getAuthorsList(oVersion) for author in authors: email = author.getEmail() if email not in ccList: ccList.append(email) with open(UPCOMING_FILE, 'w') as upcoming, \ open(UPCOMING_HEADER, 'r') as fd_header: header = {} header['messageId'] = Git.buildMessageId() header['date'] = Git.buildDate() header['name'], header['email'] = Git.getLocalUser() header['mailinglist'] = MAILING_LIST header['expectedVersion'] = User.request("Expected new version?") header['ccList'] = ", ".join(ccList) upcoming.write(fd_header.read().format(**header).lstrip()) upcoming.write(Git.getShortlog(oVersion)) upcoming.write("\n\n# Diffstat\n\n") upcoming.write(Git.getDiffstat(oVersion)) upcoming.write("\n\n\n-- \n{}\n".format(Git.getLocalUser()[0])) system("{} {}".format(EDITOR, UPCOMING_FILE)) print("{} written".format(UPCOMING_FILE)) ================================================ FILE: docs/Makefile ================================================ # This program is free software under the terms of the GNU General Public # License. See the COPYING file which must come with this package. SOURCES = $(wildcard *.rst) HTML_TARGETS = $(patsubst %.rst,%.html,$(SOURCES)) RM = rm RST2HTML=`type rst2html >/dev/null 2>&1 && echo rst2html || echo rst2html.py` RST2MAN=`type rst2man >/dev/null 2>&1 && echo rst2man || echo rst2man.py` SPHINXBUILD = sphinx-build docs: man api html: $(HTML_TARGETS) $(HTML_TARGETS): %.html : %.rst $(RST2HTML) $? $@ manhtml: offlineimap.html offlineimapui.html offlineimap.html: offlineimap.txt offlineimap.known_issues.txt a2x -v -d manpage -D manhtml -f xhtml $< offlineimapui.html: offlineimapui.txt a2x -v -d manpage -D manhtml -f xhtml $< man: offlineimap.1 offlineimapui.7 offlineimap.1: offlineimap.txt offlineimap.known_issues.txt a2x -v -d manpage -f manpage $< offlineimapui.7: offlineimapui.txt a2x -v -d manpage -f manpage $< api: $(SPHINXBUILD) -b html -d html/doctrees doc-src html websitedoc: ./website-doc.sh releases ./website-doc.sh api ./website-doc.sh html ./website-doc.sh contrib clean: $(RM) -f $(HTML_TARGETS) $(RM) -f offlineimap.1 $(RM) -f offlineimap.7 $(RM) -f manhtml/* $(RM) -rf html/* -find . -name '*.html' -exec rm -f {} \; .PHONY: clean doc ================================================ FILE: docs/build-uploads.sh ================================================ #!/bin/sh # # vim: expandtab ts=2 : WEBSITE_UPLOADS='./website/_uploads' while true do test -d .git && break cd .. done set -e echo "make clean" make clean >/dev/null echo "make targz" make targz >/dev/null # Defined in the root Makefile. version="$(./offlineimap.py --version)" abbrev="$(git log --format='%h' HEAD~1..)" targz="../offlineimap-v${version}-${abbrev}.tar.gz" filename="offlineimap-v${version}.tar.gz" mv -v "$targz" "${WEBSITE_UPLOADS}/${filename}" cd "$WEBSITE_UPLOADS" for digest in sha1 sha256 sha512 do target="${filename}.${digest}" echo "Adding digest ${WEBSITE_UPLOADS}/${target}" "${digest}sum" "$filename" > "$target" done ================================================ FILE: docs/doc-src/API.rst ================================================ .. OfflineImap API documentation .. currentmodule:: offlineimap .. _API docs: :mod:`offlineimap's` API documentation ====================================== Within :mod:`offlineimap`, the classes :class:`OfflineImap` provides the high-level functionality. The rest of the classes should usually not needed to be touched by the user. Email repositories are represented by a :class:`offlineimap.repository.Base.BaseRepository` or derivatives (see :mod:`offlineimap.repository` for details). A folder within a repository is represented by a :class:`offlineimap.folder.Base.BaseFolder` or any derivative from :mod:`offlineimap.folder`. This page contains the main API overview of OfflineImap |release|. OfflineImap can be imported as:: from offlineimap import OfflineImap :mod:`offlineimap` -- The OfflineImap module ============================================= .. module:: offlineimap .. autoclass:: offlineimap.OfflineImap(cmdline_opts = None) :members: :inherited-members: :undoc-members: :private-members: :class:`offlineimap.account` ============================ An :class:`accounts.Account` connects two email repositories that are to be synced. It comes in two flavors, normal and syncable. .. autoclass:: offlineimap.accounts.Account .. autoclass:: offlineimap.accounts.SyncableAccount :members: :inherited-members: .. autodata:: ui Contains the current :mod:`offlineimap.ui`, and can be used for logging etc. :exc:`OfflineImapError` -- A Notmuch execution error -------------------------------------------------------- .. autoexception:: offlineimap.error.OfflineImapError :members: This exception inherits directly from :exc:`Exception` and is raised on errors during the offlineimap execution. It has an attribute `severity` that denotes the severity level of the error. :mod:`offlineimap.globals` -- module with global variables ========================================================== Module `offlineimap.globals` provides the read-only storage for the global variables. All exported module attributes can be set manually, but this practice is highly discouraged and shouldn't be used. However, attributes of all stored variables can only be read, write access to them is denied. Currently, we have only :attr:`options` attribute that holds command-line options as returned by OptionParser. The value of :attr:`options` must be set by :func:`set_options` prior to its first use. .. automodule:: offlineimap.globals :members: .. data:: options You can access the values of stored options using the usual syntax, offlineimap.globals.options. ================================================ FILE: docs/doc-src/conf.py ================================================ # -*- coding: utf-8 -*- # # pyDNS documentation build configuration file, created by # sphinx-quickstart on Tue Feb 2 10:00:47 2010. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../..')) from offlineimap import __version__, __author__, __copyright__ # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] autoclass_content = "both" # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = u'OfflineIMAP' copyright = __copyright__ # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. #unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' #html_style = '' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['html'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. html_use_modindex = False # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'dev-doc' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'offlineimap.tex', u'OfflineIMAP Documentation', u'OfflineIMAP contributors', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} ================================================ FILE: docs/doc-src/dco.rst ================================================ .. _dco Developer's Certificate of Origin ================================= v1.1:: By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. Then, you just add a line saying:: Signed-off-by: Random J Developer This line can be automatically added by git if you run the git-commit command with the ``-s`` option. Signing can made be afterword with ``--amend -s``. Notice that you can place your own ``Signed-off-by:`` line when forwarding somebody else's patch with the above rules for D-C-O. Indeed you are encouraged to do so. Do not forget to place an in-body ``From:`` line at the beginning to properly attribute the change to its true author (see above). Also notice that a real name is used in the ``Signed-off-by:`` line. Please don't hide your real name. If you like, you can put extra tags at the end: Reported-by is used to to credit someone who found the bug that the patch attempts to fix. Acked-by says that the person who is more familiar with the area the patch attempts to modify liked the patch. Reviewed-by unlike the other tags, can only be offered by the reviewer and means that she is completely satisfied that the patch is ready for application. It is usually offered only after a detailed review. Tested-by is used to indicate that the person applied the patch and found it to have the desired effect. You can also create your own tag or use one that's in common usage such as ``Thanks-to:``, ``Based-on-patch-by:``, or ``Mentored-by:``. ================================================ FILE: docs/doc-src/index.rst ================================================ .. OfflineImap documentation master file .. _OfflineIMAP: http://www.offlineimap.org Welcome to OfflineIMAP's developer documentation ================================================ **License** :doc:`dco` (dco) **Documented APIs** .. toctree:: API repository ui .. moduleauthor:: John Goerzen, and many others. See AUTHORS and the git history for a full list. :License: This module is covered under the GNU GPL v2 (or later). ================================================ FILE: docs/doc-src/repository.rst ================================================ .. currentmodule:: offlineimap.repository :mod:`offlineimap.repository` -- Email repositories ------------------------------------------------------------ A derivative of class :class:`Base.BaseRepository` represents an email repository depending on the type of storage, possible options are: * :class:`IMAPRepository`, * :class:`MappedIMAPRepository` * :class:`GmailRepository`, * :class:`MaildirRepository`, or * :class:`LocalStatusRepository`. Which class you need depends on your account configuration. The helper class :class:`offlineimap.repository.Repository` is an *autoloader*, that returns the correct class depending on your configuration. So when you want to instanciate a new :mod:`offlineimap.repository`, you will mostly do it through this class. .. autoclass:: offlineimap.repository.Repository :members: :inherited-members: :mod:`offlineimap.repository.Base.BaseRepository` -- Representation of a mail repository ------------------------------------------------------------------------------------------ .. autoclass:: offlineimap.repository.Base.BaseRepository :members: :inherited-members: :undoc-members: .. .. note:: :meth:`foo` .. .. attribute:: Database.MODE Defines constants that are used as the mode in which to open a database. MODE.READ_ONLY Open the database in read-only mode MODE.READ_WRITE Open the database in read-write mode .. autoclass:: offlineimap.repository.IMAPRepository .. autoclass:: offlineimap.repository.MappedIMAPRepository .. autoclass:: offlineimap.repository.GmailRepository .. autoclass:: offlineimap.repository.MaildirRepository .. autoclass:: offlineimap.repository.LocalStatusRepository :mod:`offlineimap.folder` -- Basic representation of a local or remote Mail folder --------------------------------------------------------------------------------------------------------- .. autoclass:: offlineimap.folder.Base.BaseFolder :members: :inherited-members: :undoc-members: .. .. attribute:: Database.MODE Defines constants that are used as the mode in which to open a database. MODE.READ_ONLY Open the database in read-only mode MODE.READ_WRITE Open the database in read-write mode ================================================ FILE: docs/doc-src/ui.rst ================================================ :mod:`offlineimap.ui` -- A flexible logging system -------------------------------------------------------- .. currentmodule:: offlineimap.ui OfflineImap has various ui systems, that can be selected. They offer various functionalities. They must implement all functions that the :class:`offlineimap.ui.UIBase` offers. Early on, the ui must be set using :meth:`getglobalui` .. automethod:: offlineimap.ui.setglobalui .. automethod:: offlineimap.ui.getglobalui Base UI plugin ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: offlineimap.ui.UIBase.UIBase :members: :inherited-members: .. .. note:: :meth:`foo` .. .. attribute:: Database.MODE Defines constants that are used as the mode in which to open a database. MODE.READ_ONLY Open the database in read-only mode MODE.READ_WRITE Open the database in read-write mode ================================================ FILE: docs/offlineimap.known_issues.txt ================================================ * Deletions. + While in usual run the deletions are propagated. To prevent from data loss, removing a folder makes offlineimap re-sync the folder. However, propagating the removal of the whole content of a folder can happen in the two following cases: - The whole content of a folder is deleted but the folder directory still exists. - The parent directory of the folder was deleted. * SSL3 write pending. + Users enabling SSL may hit a bug about "SSL3 write pending". If so, the account(s) will stay unsynchronised from the time the bug appeared. Running OfflineIMAP again can help. We are still working on this bug. Patches or detailed bug reports would be appreciated. Please check you're running the last stable version and send us a report to the mailing list including the full log. * IDLE support is incomplete and experimental. Bugs may be encountered. - No hook exists for "run after an IDLE response". + Email will show up, but may not be processed until the next refresh cycle. - nametrans may not be supported correctly. - IMAP IDLE <-> IMAP IDLE doesn't work yet. - IDLE might stop syncing on a system suspend/resume. - IDLE may only work "once" per refresh. + If you encounter this bug, please send a report to the list! * Maildir support in Windows drive. + Maildir uses colon character (:) in message file names. Colon is however forbidden character in windows drives. There are several workarounds for that situation: . Enable file name character translation in windows registry (not tested). - . Use cygwin managed mount (not tested). - not available anymore since cygwin 1.7 . Use "maildir-windows-compatible = yes" account OfflineIMAP configuration. - That makes OfflineIMAP to use exclamation mark (!) instead of colon for storing messages. Such files can be written to windows partitions. But you will probably loose compatibility with other programs trying to read the same Maildir. + - Exclamation mark was chosen because of the note in http://docs.python.org/library/mailbox.html + - If you have some messages already stored without this option, you will have to re-sync them again * OfflineIMAP confused after system suspend. + When resuming a suspended session, OfflineIMAP does not cleanly handles the broken socket(s) if socktimeout option is not set. You should enable this option with a value like 10. * OfflineIMAP confused when mails change while in a sync. + When OfflineIMAP is syncing, some events happening since the invocation on remote or local side are badly handled. OfflineIMAP won't track for changes during the sync. * Sharing a maildir with multiple IMAP servers. + Generally a word of caution mixing IMAP repositories on the same Maildir root. You have to be careful that you *never* use the same maildir folder for 2 IMAP servers. In the best case, the folder MD5 will be different, and you will get a loop where it will upload your mails to both servers in turn (infinitely!) as it thinks you have placed new mails in the local Maildir. In the worst case, the MD5 is the same (likely) and mail UIDs overlap (likely too!) and it will fail to sync some mails as it thinks they are already existent. + I would create a new local Maildir Repository for the Personal Gmail and use a different root to be on the safe side here. You could e.g. use `~/mail/Pro' as Maildir root for the ProGmail and `~/mail/Personal' as root for the personal one. + If you then point your local mutt, or whatever MUA you use to `~/mail/' as root, it should still recognize all folders. * Edge cases with maxage causing too many messages to be synced. + All messages from at most maxage days ago (+/- a few hours, depending on timezones) are synced, but there are cases in which older messages can also be synced. This happens when a message's UID is significantly higher than those of other messages with similar dates, e.g. when messages are added to the local folder behind offlineimap's back, causing them to get assigned a new UID, or when offlineimap first syncs a pre-existing Maildir. In the latter case, it could appear as if a noticeable and random subset of old messages are synced. * Offlineimap hangs. + When having unexpected hangs it's advised to set `singlethreadperfolder' to 'yes', especially when in IMAP/IMAP mode (no maildir). * Passwords in netrc. + Offlineimap doesn't know how to retrieve passwords when more than one account is stored in the netrc file. See . * XOAUTH2 + XOAUTH2 might be a bit tricky to set up. Make sure you've followed the step to step guide in 'offlineimap.conf'. The known bugs about Gmail are tracked at . + Sometimes, you might hit one of the following error: - [imap]: xoauth2handler: response "{u'error': u'invalid_grant'}" - oauth2handler got: {u'error': u'invalid_grant'} + In such case, we had reports that generating a new refresh token from the same client ID and secret can help. + .Google documentation on "invalid_grant" ---- When you try to use a refresh token, the following returns you an invalid_grant error: - Your server's clock is not in sync with network time protocol - NTP. - The refresh token limit has been exceeded. ---- + .Token expiration ---- It is possible that a granted token might no longer work. A token might stop working for one of these reasons: - The user has revoked access. - The token has not been used for six months. - The user changed passwords and the token contains Gmail scopes. - The user account has exceeded a certain number of token requests. There is currently a limit of 50 refresh tokens per user account per client. If the limit is reached, creating a new token automatically invalidates the oldest token without warning. This limit does not apply to service accounts. ---- + See and to know more. * "does not have message with UID" with Microsoft servers + `ERROR: IMAP server 'Server ### Remote' does not have a message with UID 'xxx'` + Microsoft IMAP servers are not compliant with the RFC. It is currently required to folderfilter some faulting folders. See http://www.offlineimap.org/doc/FAQ.html#exchange-and-office365 for a detailed list. ================================================ FILE: docs/offlineimap.txt ================================================ offlineimap(1) ============== NAME ---- offlineimap - Synchronize mailboxes and Maildirs both ways or one either way. SYNOPSIS -------- [verse] 'offlineimap' (options) DESCRIPTION ----------- Synchronize the accounts configured in the configuration file via IMAP. Each account has two sides. One of the side must be an IMAP server. The other side can either be a Maildir or another IMAP server. Works with Python 2. OPTIONS ------- -h:: --help:: Display summary of options. --version:: Output version. -V:: Output offlineimap version and additional imaplib2 information. --dry-run:: Run in dry run mode. + Do not actually modify any store but check and print what synchronization actions would be taken if a sync would be performed. It will not precisely give the exact information what will happen. If e.g. we need to create a folder, it merely outputs 'Would create folder X', but not how many and which mails it would transfer. --info:: Output information on the configured email repositories. + Useful for debugging and bug reporting. Use in conjunction with the `-a' option to limit the output to a single account. This mode will prevent any actual sync to occur and exits after it output the debug information. -1:: Limit multithreading operations and run solely a single-thread sync. + This effectively sets the 'maxsyncaccounts' and all 'maxconnections' configuration file variables to '1' (the number). -P :: Set OfflineIMAP into profile mode. + The program will create DIR (it must not already exist). As it runs, Python profiling information about each thread is logged into profiledir. Please note: This option is present for debugging and optimization only, and should NOT be used unless you have a specific reason to do so. It will significantly decrease program performance, may reduce reliability, and can generate huge amounts of data. This option implies the `-1' option. -a :: Overrides the accounts section in the config file. + Allows one to specify a particular account or set of accounts to sync without having to edit the config file. -c :: Specifies a configuration file to use. -d :: Enables debugging for OfflineIMAP. + This is useful if you are to track down a malfunction or figure out what is going on under the hood. This option requires one or more debugtypes, separated by commas. These define what exactly will be debugged, and so far include options: "imap", "thread", "maildir" or "ALL". The imap option will enable IMAP protocol stream and parsing debugging. Note that the output may contain passwords, so take care to remove that from the debugging output before sending it to anyone else. The maildir option will enable debugging for certain Maildir operations. The use of any debug option (unless "thread" is included), implies the single-thread option `-1'. -l :: Send logs to . -s:: Send logs to syslog. -f :: Only sync the specified folders. + The folder names are the untranslated foldernames of the remote repository. This command-line option overrides any 'folderfilter' and 'folderincludes' options in the configuration file. -k <[section:]option=value:: Override any configuration file option. + If "section" is omitted, it defaults to "general". Any underscores in the section name are replaced with spaces: for instance, to override option "autorefresh" in the "[Account Personal]" section in the config file one would use `-k Account_Personal:autorefresh=30'. Repeat this option as much as necessary to redefine multiple options. -o:: Run only once. + Ignore any autorefresh setting in the configuration file. -q:: Run only quick synchronizations. + Ignore any flag updates on IMAP servers. If a flag on the remote IMAP changes, and we have the message locally, it will be left untouched in a quick run. This option is ignored if maxage is set. -u :: Specifies an alternative user interface to use. + This overrides the default specified in the configuration file. The UI specified with `-u' will be forced to be used, even if checks determine that it is not usable. Possible interface choices are: quiet, basic, syslog, ttyui, blinkenlights, machineui. --delete-folder:: Delete a folder on the remote repository. + Only one account must be specified/configured for this feature to work or you must provide one account with -a. The folder name must be provided with the remote separators (likely '/') in UTF-8 if utf8foldernames is enabled or in IMAP otherwise. E.g.: "Remote/folder/name". --migrate-fmd5-using-nametrans:: Migrate FMD5 hashes from versions prior to 6.3.5. + The way that FMD5 hashes are calculated was changed in version 6.3.5 (now using the nametrans folder name) introducing a regression which may lead to re-uploading all messages. Try and fix the above regression by calculating the correct FMD5 values and renaming the corresponding messages. CAUTION: Since the FMD5 part of the filename changes, this may lead to UID conflicts. Ensure to dispose a proper backup of both the cache and the Maildir before running this fix as well as verify the results using the `--dry-run' flag first. --mbnames-prune:: Remove dangling entries for removed accounts or if mbnames is not enabled/used anymore. + Internally, offlineimap build intermediate mbnames files. They are added automatically when mbnames is enabled. However, disabling accounts so they are not synced anymore does not necessarily means they should be removed from the file built by mbnames. It is required to start offlineimap with this CLI option each time accounts are removed. When run, any account not in the 'accounts' configuration option are removed in the mbnames file. + It is possible to manually remove intermediate files in '/mbnames/'. + Notice this option honors --dry-run. Synchronization Performance --------------------------- By default, we use fairly conservative settings that are safe for syncing but that might not be the best performing one. Once you got everything set up and running, you might want to look into speeding up your synchronization. Here are a couple of hints and tips on how to achieve this. 1. Synchronize more than one account. + By default we only use one connection to an IMAP server. Using 2 or even 3 speeds things up considerably in most cases. In order to synchronize more than one account concurrently, consider starting one instance of offlineimap per account. + WARNING: enabling the 'maxsyncaccounts' and 'maxconnections' options is deprecated since it's known to have race conditions. 2. Use folderfilters. + The quickest sync is a sync that can ignore some folders. I sort my inbox into monthly folders, and ignore every folder that is more than 2-3 months old, this lets me only inspect a fraction of my Mails on every sync. If you haven't done this yet, do it :). See the 'folderfilter' section in 'offlineimap.conf'. 3. The sqlite cache. + OfflineImap caches the state of the synchronisation to e.g. be able to determine if a mail has been added or deleted on either side. + The historical status cache was a plain text file that was writing out the complete file for each single new message (or even changed flag) to a temporary file. If there was plenty of files in a folder this was bound to make things slow. The latest status cache is sqlite. This saves plenty of disk activity. + The historical plain status cache is not supported anymore but migrating from a very old installation using the plain text cache is still supported. In this case, you may want to delete the old cache directory in '/Account-/LocalStatus' manually (the sqlite cache stands in the 'LocalStatus-sqlite' folder). First, make sure you have run the new version of offlineimap for all your accounts so that the status cache was migrated. 4. Use quick sync. + A regular sync will request all flags and all UIDs of all mails in each folder which takes quite some time. A quick sync only compares the number of messages in a folder on the IMAP side (it will detect flag changes on the Maildir side of things though). A quick sync on my smallish account will take 7 seconds rather than 40 seconds. E.g. run a cron script that does a regular sync once a day, and does quick syncs `-q' only synchronizing the `-f INBOX' in between. 5. Turn off fsync. + In the '[general]' section you can set fsync to 'True' or 'False'. If you want to play 110% safe and wait for all operations to hit the disk before continuing, you can set this to True. If you set it to False, you lose some of that safety, trading it for speed. Security and SSL ---------------- By default, OfflineIMAP will connect using any method that 'openssl' supports, that is SSLv2, SSLv3, or TLSv1. Do note that SSLv2 is notoriously insecure and deprecated. Unfortunately, python2 does not offer easy ways to disable SSLv2. It is recommended you test your setup and make sure that the mail server does not use an SSLv2 connection. Use e.g. "openssl s_client -host mail.server -port 443" to find out the connection that is used by default. * Certificate checking + Unfortunately, by default we will not verify the certificate of an IMAP TLS/SSL server we connect to, so connecting by SSL is no guarantee against man-in-the-middle attacks. While verifying a server certificate checking the fingerprint is recommended. There is currently only one safe way to ensure that you connect to the correct server in an encrypted manner: you can specify a 'sslcacertfile' setting in your repository section of offlineimap.conf pointing to a file that contains (among others) a CA Certificate in PEM format which validating your server certificate. In this case, we will check that: 1. The server SSL certificate is validated by the CA Certificate. 2. The server host name matches the SSL certificate. 3. The server certificate is not past its expiration date. The FAQ has an entry on how to create your own certificate and CA certificate. * StartTLS + If you have not configured your account to connect via SSL anyway, OfflineImap will still attempt to set up an SSL connection via the STARTTLS function, in case the imap server supports it. + There is no certificate or fingerprint checking involved at all, when using STARTTLS (the underlying imaplib library does not support this yet). This means that you will be protected against passively listening eavesdroppers and they will not be able to see your password or email contents. However, this will not protect you from active attacks, such as Man-In-The-Middle attacks which cause you to connect to the wrong server and pretend to be your mail server. + *DO NOT RELY ON STARTTLS AS A SAFE CONNECTION GUARANTEEING THE AUTHENTICITY OF YOUR IMAP SERVER!* Unix Signals ------------ OfflineImap listens to the unix signals SIGUSR1, SIGUSR2, SIGTERM, SIGINT, SIGHUP, SIGQUIT. * If sent a SIGUSR1 it will abort any current (or next future) sleep of all accounts that are configured to 'autorefresh'. In effect, this will trigger a full sync of all accounts to be performed as soon as possible. * If sent a SIGUSR2 or SIGABRT, it will stop 'autorefresh' mode for all accounts. That is, accounts will abort any current sleep and will exit after a currently running synchronization has finished. This signal can be used to gracefully exit out of a running offlineimap "daemon". * SIGTERM, SIGINT, SIGHUP are all treated to gracefully terminate as soon as possible. This means it will finish syncing the current folder in each account, close keep alive connections, remove locks on the accounts and exit. + It may take up to 10 seconds, if autorefresh option is used. + More than one SIGTERM will behave like SIGQUIT. * If sent SIGQUIT, dumps stack traces for all threads and tries to dump process core. Known Issues ------------ include::./offlineimap.known_issues.txt[] Main authors ------------ John Goerzen, Sebastian Spaetz, Eygene Ryabinkin, Nicolas Sebrecht. See Also -------- offlineimapui(7), openssl(1), signal(7), sqlite3(1). http://www.offlineimap.org ================================================ FILE: docs/offlineimapui.txt ================================================ offlineimapui(7) ================ NAME ---- offlineimapui - The User Interfaces DESCRIPTION ----------- OfflineIMAP comes with different UIs, each aiming its own purpose. TTYUI ------ TTYUI interface is for people running in terminals. It prints out basic status messages and is generally friendly to use on a console or xterm. Basic ------ Basic is designed for situations in which OfflineIMAP will be run non-attended and the status of its execution will be logged. This user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file options. For example, it will not print periodic sleep announcements and tends to be a tad less verbose, in general. Blinkenlights ------------- Blinkenlights is an interface designed to be sleek, fun to watch, and informative of the overall picture of what OfflineIMAP is doing. Blinkenlights contains a row of "LEDs" with command buttons and a log. The log shows more detail about what is happening and is color-coded to match the color of the lights. Each light in the Blinkenlights interface represents a thread of execution -- that is, a particular task that OfflineIMAP is performing right now. The colors indicate what task the particular thread is performing, and are as follows: * Black indicates that this light's thread has terminated; it will light up again later when new threads start up. So, black indicates no activity. * Red (Meaning 1) is the color of the main program's thread, which basically does nothing but monitor the others. It might remind you of HAL 9000 in 2001. * Gray indicates that the thread is establishing a new connection to the IMAP server. * Purple is the color of an account synchronization thread that is monitoring the progress of the folders in that account (not generating any I/O). * Cyan indicates that the thread is syncing a folder. * Green means that a folder's message list is being loaded. * Blue is the color of a message synchronization controller thread. * Orange indicates that an actual message is being copied. (We use fuchsia for fake messages.) * Red (meaning 2) indicates that a message is being deleted. * Yellow / bright orange indicates that message flags are being added. * Pink / bright red indicates that message flags are being removed. * Red / Black Flashing corresponds to the countdown timer that runs between synchronizations. The name of this interfaces derives from a bit of computer history. Eric Raymond's Jargon File defines blinkenlights, in part, as: Front-panel diagnostic lights on a computer, esp. a dinosaur. Now that dinosaurs are rare, this term usually refers to status lights on a modem, network hub, or the like. This term derives from the last word of the famous blackletter-Gothic sign in mangled pseudo-German that once graced about half the computer rooms in the English-speaking world. One version ran in its entirety as follows: ACHTUNG! ALLES LOOKENSPEEPERS! Das computermachine ist nicht fuer gefingerpoken und mittengrabben. Ist easy schnappen der springenwerk, blowenfusen und poppencorken mit spitzensparken. Ist nicht fuer gewerken bei das dumpkopfen. Das rubbernecken sichtseeren keepen das cotten-pickenen hans in das pockets muss; relaxen und watchen das blinkenlichten. Quiet ----- It will output nothing except errors and serious warnings. Like Basic, this user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file options. Syslog ------ Syslog is designed for situations where OfflineIMAP is run as a daemon (e.g., as a systemd --user service), but errors should be forwarded to the system log. Like Basic, this user interface is not capable of reading a password from the keyboard; account passwords must be specified using one of the configuration file options. MachineUI --------- MachineUI generates output in a machine-parsable format. It is designed for other programs that will interface to OfflineIMAP. See Also -------- offlineimap(1) ================================================ FILE: docs/rfcs/README.md ================================================ All RFCs related to IMAP. TODO: Add a brief introduction here to introduce the most important RFCs. ================================================ FILE: docs/rfcs/rfc1731.IMAP4_auth.txt ================================================ Network Working Group J. Myers Request for Comments: 1731 Carnegie Mellon Category: Standards Track December 1994 IMAP4 Authentication Mechanisms Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Introduction The Internet Message Access Protocol, Version 4 [IMAP4] contains the AUTHENTICATE command, for identifying and authenticating a user to an IMAP4 server and for optionally negotiating a protection mechanism for subsequent protocol interactions. This document describes several authentication mechanisms for use by the IMAP4 AUTHENTICATE command. 2. Kerberos version 4 authentication mechanism The authentication type associated with Kerberos version 4 is "KERBEROS_V4". The data encoded in the first ready response contains a random 32-bit number in network byte order. The client should respond with a Kerberos ticket and an authenticator for the principal "imap.hostname@realm", where "hostname" is the first component of the host name of the server with all letters in lower case and where "realm" is the Kerberos realm of the server. The encrypted checksum field included within the Kerberos authenticator should contain the server provided 32-bit number in network byte order. Upon decrypting and verifying the ticket and authenticator, the server should verify that the contained checksum field equals the original server provided random 32-bit number. Should the verification be successful, the server must add one to the checksum and construct 8 octets of data, with the first four octets containing the incremented checksum in network byte order, the fifth octet containing a bit-mask specifying the protection mechanisms supported by the server, and the sixth through eighth octets containing, in Myers [Page 1] RFC 1731 IMAP4 Authentication Mechanisms December 1994 network byte order, the maximum cipher-text buffer size the server is able to receive. The server must encrypt the 8 octets of data in the session key and issue that encrypted data in a second ready response. The client should consider the server authenticated if the first four octets the un-encrypted data is equal to one plus the checksum it previously sent. The client must construct data with the first four octets containing the original server-issued checksum in network byte order, the fifth octet containing the bit-mask specifying the selected protection mechanism, the sixth through eighth octets containing in network byte order the maximum cipher-text buffer size the client is able to receive, and the following octets containing a user name string. The client must then append from one to eight octets so that the length of the data is a multiple of eight octets. The client must then PCBC encrypt the data with the session key and respond to the second ready response with the encrypted data. The server decrypts the data and verifies the contained checksum. The username field identifies the user for whom subsequent IMAP operations are to be performed; the server must verify that the principal identified in the Kerberos ticket is authorized to connect as that user. After these verifications, the authentication process is complete. The protection mechanisms and their corresponding bit-masks are as follows: 1 No protection mechanism 2 Integrity (krb_mk_safe) protection 4 Privacy (krb_mk_priv) protection EXAMPLE: The following are two Kerberos version 4 login scenarios (note that the line breaks in the sample authenticators are for editorial clarity and are not in real authenticators) S: * OK IMAP4 Server C: A001 AUTHENTICATE KERBEROS_V4 S: + AmFYig== C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh S: + or//EoAADZI= C: DiAF5A4gA+oOIALuBkAAmw== S: A001 OK Kerberos V4 authentication successful Myers [Page 2] RFC 1731 IMAP4 Authentication Mechanisms December 1994 S: * OK IMAP4 Server C: A001 AUTHENTICATE KERBEROS_V4 S: + gcfgCA== C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh S: A001 NO Kerberos V4 authentication failed 3. GSSAPI authentication mechanism The authentication type associated with all mechanisms employing the GSSAPI [RFC1508] is "GSSAPI". The first ready response issued by the server contains no data. The client should call GSS_Init_sec_context, passing in 0 for input_context_handle (initially) and a targ_name equal to output_name from GSS_Import_Name called with input_name_type of NULL and input_name_string of "SERVICE:imap@hostname" where "hostname" is the fully qualified host name of the server with all letters in lower case. The client must then respond with the resulting output_token. If GSS_Init_sec_context returns GSS_CONTINUE_NEEDED, then the client should expect the server to issue a token in a subsequent ready response. The client must pass the token to another call to GSS_Init_sec_context. If GSS_Init_sec_context returns GSS_COMPLETE, then the client should respond with any resulting output_token. If there is no output_token, the client should respond with no data. The client should then expect the server to issue a token in a subsequent ready response. The client should pass this token to GSS_Unseal and interpret the first octet of resulting cleartext as a bit-mask specifying the protection mechanisms supported by the server and the second through fourth octets as the maximum size output_message to send to the server. The client should construct data, with the first octet containing the bit-mask specifying the selected protection mechanism, the second through fourth octets containing in network byte order the maximum size output_message the client is able to receive, and the remaining octets containing a user name string. The client must pass the data to GSS_Seal with conf_flag set to FALSE, and respond with the generated output_message. The client can then consider the server authenticated. The server must issue a ready response with no data and pass the resulting client supplied token to GSS_Accept_sec_context as input_token, setting acceptor_cred_handle to NULL (for "use default credentials"), and 0 for input_context_handle (initially). If GSS_Accept_sec_context returns GSS_CONTINUE_NEEDED, the server should Myers [Page 3] RFC 1731 IMAP4 Authentication Mechanisms December 1994 return the generated output_token to the client in a ready response and pass the resulting client supplied token to another call to GSS_Accept_sec_context. If GSS_Accept_sec_context returns GSS_COMPLETE, then if an output_token is returned, the server should return it to the client in a ready response and expect a reply from the client with no data. Whether or not an output_token was returned, the server then should then construct 4 octets of data, with the first octet containing a bit-mask specifying the protection mechanisms supported by the server and the second through fourth octets containing in network byte order the maximum size output_token the server is able to receive. The server must then pass the plaintext to GSS_Seal with conf_flag set to FALSE and issue the generated output_message to the client in a ready response. The server must then pass the resulting client supplied token to GSS_Unseal and interpret the first octet of resulting cleartext as the bit-mask for the selected protection mechanism, the second through fourth octets as the maximum size output_message to send to the client, and the remaining octets as the user name. Upon verifying the src_name is authorized to authenticate as the user name, The server should then consider the client authenticated. The protection mechanisms and their corresponding bit-masks are as follows: 1 No protection mechanism 2 Integrity protection. Sender calls GSS_Seal with conf_flag set to FALSE 4 Privacy protection. Sender calls GSS_Seal with conf_flag set to TRUE 4. S/Key authentication mechanism The authentication type associated with S/Key [SKEY] is "SKEY". The first ready response issued by the server contains no data. The client responds with the user name string. The data encoded in the second ready response contains the decimal sequence number followed by a single space and the seed string for the indicated user. The client responds with the one-time-password, as either a 64-bit value in network byte order or encoded in the "six English words" format. Upon successful verification of the one-time-password, the server should consider the client authenticated. Myers [Page 4] RFC 1731 IMAP4 Authentication Mechanisms December 1994 S/Key authentication does not provide for any protection mechanisms. EXAMPLE: The following are two S/Key login scenarios. S: * OK IMAP4 Server C: A001 AUTHENTICATE SKEY S: + C: bW9yZ2Fu S: + OTUgUWE1ODMwOA== C: Rk9VUiBNQU5OIFNPT04gRklSIFZBUlkgTUFTSA== S: A001 OK S/Key authentication successful S: * OK IMAP4 Server C: A001 AUTHENTICATE SKEY S: + C: c21pdGg= S: + OTUgUWE1ODMwOA== C: BsAY3g4gBNo= S: A001 NO S/Key authentication failed 5. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", RFC 1730, University of Washington, December 1994. [RFC1508] Linn, J., "Generic Security Service Application Program Interface", RFC 1508, Geer Zolot Associates, September 1993. [SKEY] Haller, Neil M. "The S/Key One-Time Password System", Bellcore, Morristown, New Jersey, October 1993, thumper.bellcore.com:pub/nmh/docs/ISOC.symp.ps Myers [Page 5] RFC 1731 IMAP4 Authentication Mechanisms December 1994 6. Security Considerations Security issues are discussed throughout this memo. 7. Author's Address John G. Myers Carnegie-Mellon University 5000 Forbes Ave. Pittsburgh PA, 15213-3890 EMail: jgm+@cmu.edu Myers [Page 6] ================================================ FILE: docs/rfcs/rfc1732.compatibiliy_IMAP2-IMAP2bis.txt ================================================ Network Working Group M. Crispin Request for Comments: 1732 University of Washington Category: Informational December 1994 IMAP4 COMPATIBILITY WITH IMAP2 AND IMAP2BIS Status of this Memo This memo provides information for the Internet community. This memo does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Introduction This is a summary of hints and recommendations to enable an IMAP4 implementation to interoperate with implementations that conform to earlier specifications. None of these hints and recommendations are required by the IMAP4 specification; implementors must decide for themselves whether they want their implementation to fail if it encounters old software. IMAP4 has been designed to be upwards compatible with earlier specifications. For the most part, IMAP4 facilities that were not in earlier specifications should be invisible to clients unless the client asks for the facility. In some cases, older servers may support some of the capabilities listed as being "new in IMAP4" as experimental extensions to the IMAP2 protocol described in RFC 1176. This information may not be complete; it reflects current knowledge of server and client implementations as well as "folklore" acquired in the evolution of the protocol. Crispin [Page 1] RFC 1732 IMAP4 - Compatibility December 1994 IMAP4 client interoperability with old servers In general, a client should be able to discover whether an IMAP2 server supports a facility by trial-and-error; if an attempt to use a facility generates a BAD response, the client can assume that the server does not support the facility. A quick way to check whether a server implementation supports the IMAP4 specification is to try the CAPABILITY command. An OK response that includes the IMAP4 capability value indicates a server that supports IMAP4; a BAD response or one without the IMAP4 capability value indicates an older server. The following is a list of facilities that are only in IMAP4, and suggestions for how new clients might interoperate with old servers: CAPABILITY command A BAD response to this command indicates that the server implements IMAP2 (or IMAP2bis) and not IMAP4. AUTHENTICATE command. Use the LOGIN command. LSUB and LIST commands Try the RFC 1176 FIND command. * in a sequence Use the number of messages in the mailbox from the EXISTS unsolicited response. SEARCH extensions (character set, additional criteria) Reformulate the search request using only the searching options listed in search_old in the IMAP4 grammar. This may entail doing multiple searches to achieve the desired results. BODYSTRUCTURE fetch data item Try to fetch the non-extensible BODY data item. body section number 0 Fetch the entire message and extract the header. RFC822.HEADER.LINES and RFC822.HEADER.LINES.NOT fetch data items Use RFC822.HEADER and remove the unwanted information. BODY.PEEK[section], RFC822.PEEK, and RFC822.TEXT.PEEK fetch data items Use the corresponding non-PEEK versions and manually clear the \Seen flag as necessary. Crispin [Page 2] RFC 1732 IMAP4 - Compatibility December 1994 UID fetch data item and the UID commands No equivalent capabilitity exists in older servers. FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items Use the corresponding non-SILENT versions and ignore the untagged FETCH responses which com eback. The following IMAP4 facilities were introduced in the experimental IMAP2bis revisions to RFC-1176, and may be present in a server that does not support the CAPABILITY command: CREATE, DELETE, and RENAME commands To test whether these commands are present, try a CREATE INBOX command. If the response is NO, these commands are supported by the server. If the response is BAD, they are not. Older servers without the CREATE capability may sup- port implicit creation of a mailbox by a COPY command with a non-existant name as the destination. APPEND command To test whether this command is present, try to append a zero-length stream to a mailbox name that is known not to exist (or at least, highly unlikely to exist) on the remote system. SUBSCRIBE and UNSUBSCRIBE commands Try the form of these commands with the optional MAILBOX keyword. EXAMINE command Use the SELECT command instead. flags and internal date argument to APPEND command Try the APPEND without any flag list and internal date argu- ments. BODY, BODY[section], and FULL fetch data items Use RFC822.TEXT and ALL instead. Server does not support MIME. PARTIAL command Use the appropriate FETCH command and ignore the unwanted data. IMAP4 client implementations must accept all responses and data for- mats documented in the IMAP4 specification, including those labeled Crispin [Page 3] RFC 1732 IMAP4 - Compatibility December 1994 as obsolete. This includes the COPY and STORE unsolicited responses and the old format of dates and times. In particular, client imple- mentations must not treat a date/time as a fixed format string; nor may they assume that the time begins at a particular octet. IMAP4 client implementations must not depend upon the presence of any server extensions that are not in the base IMAP4 specification. The experimental IMAP2bis version specified that the TRYCREATE spe- cial information token is sent as a separate unsolicited OK response instead of inside the NO response. The FIND BBOARDS, FIND ALL.BBOARDS, and BBOARD commands of RFC 1176 are removed from IMAP4. There is no equivalent to the bboard com- mands, which provided a separate namespace with implicit restrictions on what may be done in that namespace. Older server implementations may automatically create the destination mailbox on COPY if that mailbox does not already exist. This was how a new mailbox was created in older specifications. If the server does not support the CREATE command (see above for how to test for this), it will probably create a mailbox on COPY. Older server implementations may not preserve flags or internal dates on COPY. Some server implementations may not permit the preservation of certain flags on COPY or their setting with APPEND as site policy. Crispin [Page 4] RFC 1732 IMAP4 - Compatibility December 1994 IMAP4 server interoperability with old clients In general, there should be no interoperation problem between a server conforming to the IMAP4 specification and a well-written client that conforms to an earlier specification. Known problems are noted below: Poor wording in the description of the CHECK command in earlier specifications implied that a CHECK command is the way to get the current number of messages in the mailbox. This is incorrect. A CHECK command does not necessarily result in an EXISTS response. Clients must remember the most recent EXISTS value sent from the server, and should not generate unnecessary CHECK commands. An incompatibility exists with COPY in IMAP4. COPY in IMAP4 servers does not automatically create the destination mailbox if that mailbox does not already exist. This may cause problems with old clients that expect automatic mailbox creation in COPY. The PREAUTH unsolicited response is new in IMAP4. It is highly unlikely that an old client would ever see this response. The format of dates and times has changed due to the impending end of the century. Clients that fail to accept a four-digit year or a signed four-digit timezone value will not work properly with IMAP4. An incompatibility exists with the use of "\" in quoted strings. This is best avoided by using literals instead of quoted strings if "\" or <"> is embedded in the string. Security Considerations Security issues are not discussed in this memo. Author's Address: Mark R. Crispin Networks and Distributed Computing, JE-30 University of Washington Seattle, WA 98195 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin [Page 5] ================================================ FILE: docs/rfcs/rfc1733.models_in_IMAP4.txt ================================================ Network Working Group M. Crispin Request for Comments: 1733 University of Washington Category: Informational December 1994 DISTRIBUTED ELECTRONIC MAIL MODELS IN IMAP4 Status of this Memo This memo provides information for the Internet community. This memo does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Distributed Electronic Mail Models There are three fundamental models of client/server email: offline, online, and disconnected use. IMAP4 can be used in any one of these three models. The offline model is the most familiar form of client/server email today, and is used by protocols such as POP-3 (RFC 1225) and UUCP. In this model, a client application periodically connects to a server. It downloads all the pending messages to the client machine and deletes these from the server. Thereafter, all mail processing is local to the client. This model is store-and-forward; it moves mail on demand from an intermediate server (maildrop) to a single destination machine. The online model is most commonly used with remote filesystem protocols such as NFS. In this model, a client application manipulates mailbox data on a server machine. A connection to the server is maintained throughout the session. No mailbox data are kept on the client; the client retrieves data from the server as is needed. IMAP4 introduces a form of the online model that requires considerably less network bandwidth than a remote filesystem protocol, and provides the opportunity for using the server for CPU or I/O intensive functions such as parsing and searching. The disconnected use model is a hybrid of the offline and online models, and is used by protocols such as PCMAIL (RFC 1056). In this model, a client user downloads some set of messages from the server, manipulates them offline, then at some later time uploads the changes. The server remains the authoritative repository of the messages. The problems of synchronization (particularly when multiple clients are involved) are handled through the means of unique identifiers for each message. Crispin [Page 1] RFC 1733 IMAP4 - Model December 1994 Each of these models have their own strengths and weaknesses: Feature Offline Online Disc ------- ------- ------ ---- Can use multiple clients NO YES YES Minimum use of server connect time YES NO YES Minimum use of server resources YES NO NO Minimum use of client disk resources NO YES NO Multiple remote mailboxes NO YES YES Fast startup NO YES NO Mail processing when not online YES NO YES Although IMAP4 has its origins as a protocol designed to accommodate the online model, it can support the other two models as well. This makes possible the creation of clients that can be used in any of the three models. For example, a user may wish to switch between the online and disconnected models on a regular basis (e.g. owing to travel). IMAP4 is designed to transmit message data on demand, and to provide the facilities necessary for a client to decide what data it needs at any particular time. There is generally no need to do a wholesale transfer of an entire mailbox or even of the complete text of a message. This makes a difference in situations where the mailbox is large, or when the link to the server is slow. More specifically, IMAP4 supports server-based RFC 822 and MIME processing. With this information, it is possible for a client to determine in advance whether it wishes to retrieve a particular message or part of a message. For example, a user connected to an IMAP4 server via a dialup link can determine that a message has a 2000 byte text segment and a 40 megabyte video segment, and elect to fetch only the text segment. In IMAP4, the client/server relationship lasts only for the duration of the TCP connection. There is no registration of clients. Except for any unique identifiers used in disconnected use operation, the client initially has no knowledge of mailbox state and learns it from the IMAP4 server when a mailbox is selected. This initial transfer is minimal; the client requests additional state data as it needs. As noted above, the choice for the location of mailbox data depends upon the model chosen. The location of message state (e.g. whether or not a message has been read or answered) is also determined by the model, and is not necessarily the same as the location of the mailbox data. For example, in the online model message state can be co- located with mailbox data; it can also be located elsewhere (on the client or on a third agent) using unique identifiers to achieve Crispin [Page 2] RFC 1733 IMAP4 - Model December 1994 common reference across sessions. The latter is particularly useful with a server that exports public data such as netnews and does not maintain per-user state. The IMAP4 protocol provides the generality to implement these different models. This is done by means of server and (especially) client configuration, and not by requiring changes to the protocol or the implementation of the protocol. Security Considerations Security issues are not discussed in this memo. Author's Address: Mark R. Crispin Networks and Distributed Computing, JE-30 University of Washington Seattle, WA 98195 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin [Page 3] ================================================ FILE: docs/rfcs/rfc1734.POP3_AUTHentication ================================================ RFC 1734 - POP3 AUTHentication command
[Docs] [txt|pdf] [draft-myers-pop3-...] [Diff1] [Diff2]

Obsoleted by: 5034 PROPOSED STANDARD

Network Working Group                                           J. Myers
Request for Comments: 1734                               Carnegie Mellon
Category: Standards Track                                  December 1994


                      POP3 AUTHentication command

Status of this Memo

   This document specifies an Internet standards track protocol for the
   Internet community, and requests discussion and suggestions for
   improvements.  Please refer to the current edition of the "Internet
   Official Protocol Standards" (STD 1) for the standardization state
   and status of this protocol.  Distribution of this memo is unlimited.


1. Introduction

   This document describes the optional AUTH command, for indicating an
   authentication mechanism to the server, performing an authentication
   protocol exchange, and optionally negotiating a protection mechanism
   for subsequent protocol interactions.  The authentication and
   protection mechanisms used by the POP3 AUTH command are those used by
   IMAP4.


2. The AUTH command

   AUTH mechanism

         Arguments:
             a string identifying an IMAP4 authentication mechanism,
             such as defined by [IMAP4-AUTH].  Any use of the string
             "imap" used in a server authentication identity in the
             definition of an authentication mechanism is replaced with
             the string "pop".

         Restrictions:
             may only be given in the AUTHORIZATION state

         Discussion:
             The AUTH command indicates an authentication mechanism to
             the server.  If the server supports the requested
             authentication mechanism, it performs an authentication
             protocol exchange to authenticate and identify the user.
             Optionally, it also negotiates a protection mechanism for
             subsequent protocol interactions.  If the requested
             authentication mechanism is not supported, the server



Myers                                                           [Page 1]


RFC 1734                       POP3 AUTH                   December 1994


             should reject the AUTH command by sending a negative
             response.

             The authentication protocol exchange consists of a series
             of server challenges and client answers that are specific
             to the authentication mechanism.  A server challenge,
             otherwise known as a ready response, is a line consisting
             of a "+" character followed by a single space and a BASE64
             encoded string.  The client answer consists of a line
             containing a BASE64 encoded string.  If the client wishes
             to cancel an authentication exchange, it should issue a
             line with a single "*".  If the server receives such an
             answer, it must reject the AUTH command by sending a
             negative response.

             A protection mechanism provides integrity and privacy
             protection to the protocol session.  If a protection
             mechanism is negotiated, it is applied to all subsequent
             data sent over the connection.  The protection mechanism
             takes effect immediately following the CRLF that concludes
             the authentication exchange for the client, and the CRLF of
             the positive response for the server.  Once the protection
             mechanism is in effect, the stream of command and response
             octets is processed into buffers of ciphertext.  Each
             buffer is transferred over the connection as a stream of
             octets prepended with a four octet field in network byte
             order that represents the length of the following data.
             The maximum ciphertext buffer length is defined by the
             protection mechanism.

             The server is not required to support any particular
             authentication mechanism, nor are authentication mechanisms
             required to support any protection mechanisms.  If an AUTH
             command fails with a negative response, the session remains
             in the AUTHORIZATION state and client may try another
             authentication mechanism by issuing another AUTH command,
             or may attempt to authenticate by using the USER/PASS or
             APOP commands.  In other words, the client may request
             authentication types in decreasing order of preference,
             with the USER/PASS or APOP command as a last resort.

             Should the client successfully complete the authentication
             exchange, the POP3 server issues a positive response and
             the POP3 session enters the TRANSACTION state.

         Possible Responses:
             +OK maildrop locked and ready
             -ERR authentication exchange failed



Myers                                                           [Page 2]


RFC 1734                       POP3 AUTH                   December 1994



         Examples:
             S: +OK POP3 server ready
             C: AUTH KERBEROS_V4
             S: + AmFYig==
             C: BAcAQU5EUkVXLkNNVS5FRFUAOCAsho84kLN3/IJmrMG+25a4DT
                +nZImJjnTNHJUtxAA+o0KPKfHEcAFs9a3CL5Oebe/ydHJUwYFd
                WwuQ1MWiy6IesKvjL5rL9WjXUb9MwT9bpObYLGOKi1Qh
             S: + or//EoAADZI=
             C: DiAF5A4gA+oOIALuBkAAmw==
             S: +OK Kerberos V4 authentication successful
                ...
             C: AUTH FOOBAR
             S: -ERR Unrecognized authentication type

              Note: the line breaks in the first client answer  are
              for editorial clarity and are not in real authentica-
              tors.

































Myers                                                           [Page 3]


RFC 1734                       POP3 AUTH                   December 1994


3. Formal Syntax

   The following syntax specification uses the augmented Backus-Naur
   Form (BNF) notation as specified in RFC 822.

   Except as noted otherwise, all alphabetic characters are case-
   insensitive.  The use of upper or lower case characters to define
   token strings is for editorial clarity only.  Implementations MUST
   accept these strings in a case-insensitive fashion.

   ATOM_CHAR       ::= <any CHAR except atom_specials>

   atom_specials   ::= "(" / ")" / "{" / SPACE / CTLs / "%" / "*" /
                       <"> / "\"

   auth            ::= "AUTH" 1*(SPACE / TAB) auth_type *(CRLF base64)
                       CRLF

   auth_type       ::= 1*ATOM_CHAR

   base64          ::= *(4base64_CHAR) [base64_terminal]

   base64_char     ::= "A" / "B" / "C" / "D" / "E" / "F" / "G" / "H" /
           "I" / "J" / "K" / "L" / "M" / "N" / "O" / "P" /
                       "Q" / "R" / "S" / "T" / "U" / "V" / "W" / "X" /
                       "Y" / "Z" /
                       "a" / "b" / "c" / "d" / "e" / "f" / "g" / "h" /
                       "i" / "j" / "k" / "l" / "m" / "n" / "o" / "p" /
                       "q" / "r" / "s" / "t" / "u" / "v" / "w" / "x" /
                       "y" / "z" /
                       "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" /
                       "8" / "9" / "+" / "/"
                       ;; Case-sensitive

   base64_terminal ::= (2base64_char "==") / (3base64_char "=")

   CHAR            ::= <any 7-bit US-ASCII character except NUL,
                        0x01 - 0x7f>

   continue_req    ::= "+" SPACE base64 CRLF

   CR              ::= <ASCII CR, carriage return, 0x0C>

   CRLF            ::= CR LF

   CTL             ::= <any ASCII control character and DEL,
                        0x00 - 0x1f, 0x7f>




Myers                                                           [Page 4]


RFC 1734                       POP3 AUTH                   December 1994


   LF              ::= <ASCII LF, line feed, 0x0A>

   SPACE           ::= <ASCII SP, space, 0x20>

   TAB             ::= <ASCII HT, tab, 0x09>



4. References

   [IMAP4-AUTH]  Myers, J., "IMAP4 Authentication Mechanisms", RFC 1731,
   Carnegie Mellon, December 1994.



5. Security Considerations

   Security issues are discussed throughout this memo.



6. Author's Address

   John G. Myers
   Carnegie-Mellon University
   5000 Forbes Ave
   Pittsburgh, PA 15213

   EMail: jgm+@cmu.edu






















Myers                                                           [Page 5]

Html markup produced by rfcmarkup 1.111, available from https://tools.ietf.org/tools/rfcmarkup/ ================================================ FILE: docs/rfcs/rfc2061.compatibility_IMAP4-IMAP2bis.txt ================================================ Network Working Group M. Crispin Request for Comments: 2061 University of Washington Category: Informational December 1996 IMAP4 COMPATIBILITY WITH IMAP2BIS Status of this Memo This memo provides information for the Internet community. This memo does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Introduction The Internet Message Access Protocol (IMAP) has been through several revisions and variants in its 10-year history. Many of these are either extinct or extremely rare; in particular, several undocumented variants and the variants described in RFC 1064, RFC 1176, and RFC 1203 fall into this category. One variant, IMAP2bis, is at the time of this writing very common and has been widely distributed with the Pine mailer. Unfortunately, there is no definite document describing IMAP2bis. This document is intended to be read along with RFC 1176 and the most recent IMAP4 specification (RFC 2060) to assist implementors in creating an IMAP4 implementation to interoperate with implementations that conform to earlier specifications. Nothing in this document is required by the IMAP4 specification; implementors must decide for themselves whether they want their implementation to fail if it encounters old software. At the time of this writing, IMAP4 has been updated from the version described in RFC 1730. An implementor who wishes to interoperate with both RFC 1730 and RFC 2060 should refer to both documents. This information is not complete; it reflects current knowledge of server and client implementations as well as "folklore" acquired in the evolution of the protocol. It is NOT a description of how to interoperate with all variants of IMAP, but rather with the old variant that is most likely to be encountered. For detailed information on interoperating with other old variants, refer to RFC 1732. IMAP4 client interoperability with IMAP2bis servers A quick way to check whether a server implementation supports the IMAP4 specification is to try the CAPABILITY command. An OK response will indicate which variant(s) of IMAP4 are supported by the server. Crispin Informational [Page 1] RFC 2061 IMAP4 Compatibility December 1996 If the client does not find any of its known variant in the response, it should treat the server as IMAP2bis. A BAD response indicates an IMAP2bis or older server. Most IMAP4 facilities are in IMAP2bis. The following exceptions exist: CAPABILITY command The absense of this command indicates IMAP2bis (or older). AUTHENTICATE command. Use the LOGIN command. LSUB, SUBSCRIBE, and UNSUBSCRIBE commands No direct functional equivalent. IMAP2bis had a concept called "bboards" which is not in IMAP4. RFC 1176 supported these with the BBOARD and FIND BBOARDS commands. IMAP2bis augmented these with the FIND ALL.BBOARDS, SUBSCRIBE BBOARD, and UNSUBSCRIBE BBOARD commands. It is recommended that none of these commands be implemented in new software, including servers that support old clients. LIST command Use the command FIND ALL.MAILBOXES, which has a similar syn- tax and response to the FIND MAILBOXES command described in RFC 1176. The FIND MAILBOXES command is unlikely to produce useful information. * in a sequence Use the number of messages in the mailbox from the EXISTS unsolicited response. SEARCH extensions (character set, additional criteria) Reformulate the search request using only the RFC 1176 syn- tax. This may entail doing multiple searches to achieve the desired results. BODYSTRUCTURE fetch data item Use the non-extensible BODY data item. body sections HEADER, TEXT, MIME, HEADER.FIELDS, HEADER.FIELDS.NOT Use body section numbers only. BODY.PEEK[section] Use BODY[section] and manually clear the \Seen flag as necessary. Crispin Informational [Page 2] RFC 2061 IMAP4 Compatibility December 1996 FLAGS.SILENT, +FLAGS.SILENT, and -FLAGS.SILENT store data items Use the corresponding non-SILENT versions and ignore the untagged FETCH responses which come back. UID fetch data item and the UID commands No functional equivalent. CLOSE command No functional equivalent. In IMAP2bis, the TRYCREATE special information token is sent as a separate unsolicited OK response instead of inside the NO response. IMAP2bis is ambiguous about whether or not flags or internal dates are preserved on COPY. It is impossible to know what behavior is supported by the server. IMAP4 server interoperability with IMAP2bis clients The only interoperability problem between an IMAP4 server and a well-written IMAP2bis client is an incompatibility with the use of "\" in quoted strings. This is best avoided by using literals instead of quoted strings if "\" or <"> is embedded in the string. Security Considerations Security issues are not discussed in this memo. Author's Address Mark R. Crispin Networks and Distributed Computing University of Washington 4545 15th Aveneue NE Seattle, WA 98105-4527 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin Informational [Page 3] ================================================ FILE: docs/rfcs/rfc2086.IMAP4_ACL_extension.txt ================================================ Network Working Group J. Myers Request for Comments: 2086 Carnegie Mellon Category: Standards Track January 1997 IMAP4 ACL extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Abstract The ACL extension of the Internet Message Access Protocol [IMAP4] permits access control lists to be manipulated through the IMAP protocol. Table of Contents 1. Abstract............................................... 1 2. Conventions Used in this Document...................... 1 3. Introduction and Overview.............................. 2 4. Commands............................................... 3 4.1. SETACL................................................. 3 4.2. DELETEACL.............................................. 4 4.3. GETACL................................................. 4 4.4. LISTRIGHTS............................................. 4 4.5. MYRIGHTS............................................... 5 5. Responses.............................................. 5 5.1. ACL.................................................... 5 5.2. LISTRIGHTS............................................. 6 5.3. MYRIGHTS............................................... 6 6. Formal Syntax.......................................... 6 7. References............................................. 7 8. Security Considerations................................ 7 9. Author's Address....................................... 8 2. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. Myers Standards Track [Page 1] RFC 2086 ACL extension January 1997 3. Introduction and Overview The ACL extension is present in any IMAP4 implementation which returns "ACL" as one of the supported capabilities to the CAPABILITY command. An access control list is a set of pairs. Identifier is a US-ASCII string. The identifier anyone is reserved to refer to the universal identity (all authentications, including anonymous). All user name strings accepted by the LOGIN or AUTHENTICATE commands to authenticate to the IMAP server are reserved as identifiers for the corresponding user. Identifiers starting with a dash ("-") are reserved for "negative rights", described below. All other identifier strings are interpreted in an implementation- defined manner. Rights is a string listing a (possibly empty) set of alphanumeric characters, each character listing a set of operations which is being controlled. Letters are reserved for ``standard'' rights, listed below. The set of standard rights may only be extended by a standards-track document. Digits are reserved for implementation or site defined rights. The currently defined standard rights are: l - lookup (mailbox is visible to LIST/LSUB commands) r - read (SELECT the mailbox, perform CHECK, FETCH, PARTIAL, SEARCH, COPY from mailbox) s - keep seen/unseen information across sessions (STORE SEEN flag) w - write (STORE flags other than SEEN and DELETED) i - insert (perform APPEND, COPY into mailbox) p - post (send mail to submission address for mailbox, not enforced by IMAP4 itself) c - create (CREATE new sub-mailboxes in any implementation-defined hierarchy) d - delete (STORE DELETED flag, perform EXPUNGE) a - administer (perform SETACL) An implementation may tie rights together or may force rights to always or never be granted to particular identifiers. For example, in an implementation that uses unix mode bits, the rights "wisd" are tied, the "a" right is always granted to the owner of a mailbox and is never granted to another user. If rights are tied in an implementation, the implementation must be conservative in granting rights in response to SETACL commands--unless all rights in a tied set are specified, none of that set should be included in the ACL entry for that identifier. A client may discover the set of rights which may be granted to a given identifier in the ACL for a given mailbox by using the LISTRIGHTS command. Myers Standards Track [Page 2] RFC 2086 ACL extension January 1997 It is possible for multiple identifiers in an access control list to apply to a given user (or other authentication identity). For example, an ACL may include rights to be granted to the identifier matching the user, one or more implementation-defined identifiers matching groups which include the user, and/or the identifier "anyone". How these rights are combined to determine the user's access is implementation-defined. An implementation may choose, for example, to use the union of the rights granted to the applicable identifiers. An implementation may instead choose, for example, to only use those rights granted to the most specific identifier present in the ACL. A client may determine the set of rights granted to the logged-in user for a given mailbox by using the MYRIGHTS command. When an identifier in an ACL starts with a dash ("-"), that indicates that associated rights are to be removed from the identifier that is prefixed by the dash. For example, if the identifier "-fred" is granted the "w" right, that indicates that the "w" right is to be removed from users matching the identifier "fred". Implementations need not support having identifiers which start with a dash in ACLs. 4. Commands 4.1. SETACL Arguments: mailbox name authentication identifier access right modification Data: no specific data for this command Result: OK - setacl completed NO - setacl failure: can't set acl BAD - command unknown or arguments invalid The SETACL command changes the access control list on the specified mailbox so that the specified identifier is granted permissions as specified in the third argument. The third argument is a string containing an optional plus ("+") or minus ("-") prefix, followed by zero or more rights characters. If the string starts with a plus, the following rights are added to any existing rights for the identifier. If the string starts with a minus, the following rights are removed from any existing rights for the identifier. If the string does not start with a plus or minus, the rights replace any existing rights for the identifier. Myers Standards Track [Page 3] RFC 2086 ACL extension January 1997 4.2. DELETEACL Arguments: mailbox name authentication identifier Data: no specific data for this command Result: OK - deleteacl completed NO - deleteacl failure: can't delete acl BAD - command unknown or arguments invalid The DELETEACL command removes any pair for the specified identifier from the access control list for the specified mailbox. 4.3. GETACL Arguments: mailbox name Data: untagged responses: ACL Result: OK - getacl completed NO - getacl failure: can't get acl BAD - command unknown or arguments invalid The GETACL command returns the access control list for mailbox in an untagged ACL reply. Example: C: A002 GETACL INBOX S: * ACL INBOX Fred rwipslda S: A002 OK Getacl complete 4.4. LISTRIGHTS Arguments: mailbox name authentication identifier Data: untagged responses: LISTRIGHTS Result: OK - listrights completed NO - listrights failure: can't get rights list BAD - command unknown or arguments invalid The LISTRIGHTS command takes a mailbox name and an identifier and returns information about what rights may be granted to the identifier in the ACL for the mailbox. Myers Standards Track [Page 4] RFC 2086 ACL extension January 1997 Example: C: a001 LISTRIGHTS ~/Mail/saved smith S: * LISTRIGHTS ~/Mail/saved smith la r swicd S: a001 OK Listrights completed C: a005 LISTRIGHTS archive.imap anyone S: * LISTRIGHTS archive.imap anyone "" l r s w i p c d a 0 1 2 3 4 5 6 7 8 9 4.5. MYRIGHTS Arguments: mailbox name Data: untagged responses: MYRIGHTS Result: OK - myrights completed NO - myrights failure: can't get rights BAD - command unknown or arguments invalid The MYRIGHTS command returns the set of rights that the user has to mailbox in an untagged MYRIGHTS reply. Example: C: A003 MYRIGHTS INBOX S: * MYRIGHTS INBOX rwipslda S: A003 OK Myrights complete 5. Responses 5.1. ACL Data: mailbox name zero or more identifier rights pairs The ACL response occurs as a result of a GETACL command. The first string is the mailbox name for which this ACL applies. This is followed by zero or more pairs of strings, each pair contains the identifier for which the entry applies followed by the set of rights that the identifier has. Myers Standards Track [Page 5] RFC 2086 ACL extension January 1997 5.2. LISTRIGHTS Data: mailbox name identifier required rights list of optional rights The LISTRIGHTS response occurs as a result of a LISTRIGHTS command. The first two strings are the mailbox name and identifier for which this rights list applies. Following the identifier is a string containing the (possibly empty) set of rights the identifier will always be granted in the mailbox. Following this are zero or more strings each containing a set of rights the identifier may be granted in the mailbox. Rights mentioned in the same string are tied together--either all must be granted to the identifier in the mailbox or none may be granted. The same right may not be listed more than once in the LISTRIGHTS command. 5.3. MYRIGHTS Data: mailbox name rights The MYRIGHTS response occurs as a result of a MYRIGHTS command. The first string is the mailbox name for which these rights apply. The second string is the set of rights that the client has. 6. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. Non-terminals referenced but not defined below are as defined by [IMAP4]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. Myers Standards Track [Page 6] RFC 2086 ACL extension January 1997 acl_data ::= "ACL" SPACE mailbox *(SPACE identifier SPACE rights) deleteacl ::= "DELETEACL" SPACE mailbox SPACE identifier getacl ::= "GETACL" SPACE mailbox identifier ::= astring listrights ::= "LISTRIGHTS" SPACE mailbox SPACE identifier listrights_data ::= "LISTRIGHTS" SPACE mailbox SPACE identifier SPACE rights *(SPACE rights) mod_rights ::= astring ;; +rights to add, -rights to remove ;; rights to replace myrights ::= "MYRIGHTS" SPACE mailbox myrights_data ::= "MYRIGHTS" SPACE mailbox SPACE rights rights ::= astring setacl ::= "SETACL" SPACE mailbox SPACE identifier SPACE mod_rights 7. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", RFC 1730, University of Washington, December 1994. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822. 8. Security Considerations An implementation must make sure the ACL commands themselves do not give information about mailboxes with appropriately restricted ACL's. For example, a GETACL command on a mailbox for which the user has insufficient rights should not admit the mailbox exists, much less return the mailbox's ACL. Myers Standards Track [Page 7] RFC 2086 ACL extension January 1997 9. Author's Address John G. Myers Carnegie-Mellon University 5000 Forbes Ave. Pittsburgh PA, 15213-3890 Email: jgm+@cmu.edu Myers Standards Track [Page 8] ================================================ FILE: docs/rfcs/rfc2087.IMAP4_QUOTA_extension.txt ================================================ Network Working Group J. Myers Request for Comments: 2087 Carnegie Mellon Category: Standards Track January 1997 IMAP4 QUOTA extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Abstract The QUOTA extension of the Internet Message Access Protocol [IMAP4] permits administrative limits on resource usage (quotas) to be manipulated through the IMAP protocol. Table of Contents 1. Abstract........................................... 1 2. Conventions Used in this Document.................. 1 3. Introduction and Overview.......................... 2 4. Commands........................................... 2 4.1. SETQUOTA Command................................... 2 4.2. GETQUOTA Command................................... 2 4.3. GETQUOTAROOT Command............................... 3 5. Responses.......................................... 3 5.1. QUOTA Response..................................... 3 5.2. QUOTAROOT Response................................. 4 6. Formal syntax...................................... 4 7. References......................................... 5 8. Security Considerations............................ 5 9. Author's Address................................... 5 2. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. Myers Standards Track [Page 1] RFC 2087 QUOTA January 1997 3. Introduction and Overview The QUOTA extension is present in any IMAP4 implementation which returns "QUOTA" as one of the supported capabilities to the CAPABILITY command. An IMAP4 server which supports the QUOTA capability may support limits on any number of resources. Each resource has an atom name and an implementation-defined interpretation which evaluates to an integer. Examples of such resources are: Name Interpretation STORAGE Sum of messages' RFC822.SIZE, in units of 1024 octets MESSAGE Number of messages Each mailbox has zero or more implementation-defined named "quota roots". Each quota root has zero or more resource limits. All mailboxes that share the same named quota root share the resource limits of the quota root. Quota root names do not necessarily have to match the names of existing mailboxes. 4. Commands 4.1. SETQUOTA Command Arguments: quota root list of resource limits Data: untagged responses: QUOTA Result: OK - setquota completed NO - setquota error: can't set that data BAD - command unknown or arguments invalid The SETQUOTA command takes the name of a mailbox quota root and a list of resource limits. The resource limits for the named quota root are changed to be the specified limits. Any previous resource limits for the named quota root are discarded. If the named quota root did not previously exist, an implementation may optionally create it and change the quota roots for any number of existing mailboxes in an implementation-defined manner. Myers Standards Track [Page 2] RFC 2087 QUOTA January 1997 Example: C: A001 SETQUOTA "" (STORAGE 512) S: * QUOTA "" (STORAGE 10 512) S: A001 OK Setquota completed 4.2. GETQUOTA Command Arguments: quota root Data: untagged responses: QUOTA Result: OK - getquota completed NO - getquota error: no such quota root, permission denied BAD - command unknown or arguments invalid The GETQUOTA command takes the name of a quota root and returns the quota root's resource usage and limits in an untagged QUOTA response. Example: C: A003 GETQUOTA "" S: * QUOTA "" (STORAGE 10 512) S: A003 OK Getquota completed 4.3. GETQUOTAROOT Command Arguments: mailbox name Data: untagged responses: QUOTAROOT, QUOTA Result: OK - getquota completed NO - getquota error: no such mailbox, permission denied BAD - command unknown or arguments invalid The GETQUOTAROOT command takes the name of a mailbox and returns the list of quota roots for the mailbox in an untagged QUOTAROOT response. For each listed quota root, it also returns the quota root's resource usage and limits in an untagged QUOTA response. Example: C: A003 GETQUOTAROOT INBOX S: * QUOTAROOT INBOX "" S: * QUOTA "" (STORAGE 10 512) S: A003 OK Getquota completed Myers Standards Track [Page 3] RFC 2087 QUOTA January 1997 5. Responses 5.1. QUOTA Response Data: quota root name list of resource names, usages, and limits This response occurs as a result of a GETQUOTA or GETQUOTAROOT command. The first string is the name of the quota root for which this quota applies. The name is followed by a S-expression format list of the resource usage and limits of the quota root. The list contains zero or more triplets. Each triplet conatins a resource name, the current usage of the resource, and the resource limit. Resources not named in the list are not limited in the quota root. Thus, an empty list means there are no administrative resource limits in the quota root. Example: S: * QUOTA "" (STORAGE 10 512) 5.2. QUOTAROOT Response Data: mailbox name zero or more quota root names This response occurs as a result of a GETQUOTAROOT command. The first string is the mailbox and the remaining strings are the names of the quota roots for the mailbox. Example: S: * QUOTAROOT INBOX "" S: * QUOTAROOT comp.mail.mime 6. Formal syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in RFC 822 with one exception; the delimiter used with the "#" construct is a single space (SP) and not one or more commas. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. Myers Standards Track [Page 4] RFC 2087 QUOTA January 1997 getquota ::= "GETQUOTA" SP astring getquotaroot ::= "GETQUOTAROOT" SP astring quota_list ::= "(" #quota_resource ")" quota_resource ::= atom SP number SP number quota_response ::= "QUOTA" SP astring SP quota_list quotaroot_response ::= "QUOTAROOT" SP astring *(SP astring) setquota ::= "SETQUOTA" SP astring SP setquota_list setquota_list ::= "(" 0#setquota_resource ")" setquota_resource ::= atom SP number 7. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", RFC 1730, University of Washington, December 1994. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822. 8. Security Considerations Implementors should be careful to make sure the implementation of these commands does not violate the site's security policy. The resource usage of other users is likely to be considered confidential information and should not be divulged to unauthorized persons. 9. Author's Address John G. Myers Carnegie-Mellon University 5000 Forbes Ave. Pittsburgh PA, 15213-3890 EMail: jgm+@cmu.edu Myers Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc2088.IMAP4_non_synchronizing_literals.txt ================================================ Network Working Group J. Myers Request for Comments: 2088 Carnegie Mellon Cateogry: Standards Track January 1997 IMAP4 non-synchronizing literals Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Abstract The Internet Message Access Protocol [IMAP4] contains the "literal" syntactic construct for communicating strings. When sending a literal from client to server, IMAP4 requires the client to wait for the server to send a command continuation request between sending the octet count and the string data. This document specifies an alternate form of literal which does not require this network round trip. 2. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. 3. Specification The non-synchronizing literal is added an alternate form of literal, and may appear in communication from client to server instead of the IMAP4 form of literal. The IMAP4 form of literal, used in communication from client to server, is referred to as a synchronizing literal. Non-synchronizing literals may be used with any IMAP4 server implementation which returns "LITERAL+" as one of the supported capabilities to the CAPABILITY command. If the server does not advertise the LITERAL+ capability, the client must use synchronizing literals instead. The non-synchronizing literal is distinguished from the original synchronizing literal by having a plus ('+') between the octet count and the closing brace ('}'). The server does not generate a command continuation request in response to a non-synchronizing literal, and Myers Standards Track [Page 1] RFC 2088 LITERAL January 1997 clients are not required to wait before sending the octets of a non- synchronizing literal. The protocol receiver of an IMAP4 server must check the end of every received line for an open brace ('{') followed by an octet count, a plus ('+'), and a close brace ('}') immediately preceeding the CRLF. If it finds this sequence, it is the octet count of a non- synchronizing literal and the server MUST treat the specified number of following octets and the following line as part of the same command. A server MAY still process commands and reject errors on a line-by-line basis, as long as it checks for non-synchronizing literals at the end of each line. Example: C: A001 LOGIN {11+} C: FRED FOOBAR {7+} C: fat man S: A001 OK LOGIN completed 4. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. Non-terminals referenced but not defined below are as defined by [IMAP4]. literal ::= "{" number ["+"] "}" CRLF *CHAR8 ;; Number represents the number of CHAR8 octets 6. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4", draft-crispin-imap-base-XX.txt, University of Washington, April 1996. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822. 7. Security Considerations There are no known security issues with this extension. 8. Author's Address John G. Myers Carnegie-Mellon University 5000 Forbes Ave. Pittsburgh PA, 15213-3890 Email: jgm+@cmu.edu Myers Standards Track [Page 2] ================================================ FILE: docs/rfcs/rfc2095.IMAP-POP_AUTHorize_extension.txt ================================================ Network Working Group J. Klensin Request for Comments: 2095 R. Catoe Category: Standards Track P. Krumviede MCI January 1997 IMAP/POP AUTHorize Extension for Simple Challenge/Response Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract While IMAP4 supports a number of strong authentication mechanisms as described in RFC 1731, it lacks any mechanism that neither passes cleartext, reusable passwords across the network nor requires either a significant security infrastructure or that the mail server update a mail-system-wide user authentication file on each mail access. This specification provides a simple challenge-response authentication protocol that is suitable for use with IMAP4. Since it utilizes Keyed-MD5 digests and does not require that the secret be stored in the clear on the server, it may also constitute an improvement on APOP for POP3 use as specified in RFC 1734. 1. Introduction Existing Proposed Standards specify an AUTHENTICATE mechanism for the IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is intended to be extensible; the four methods specified in [IMAP-AUTH] are all fairly powerful and require some security infrastructure to support. The base POP3 specification [POP3] also contains a lightweight challenge-response mechanism called APOP. APOP is associated with most of the risks associated with such protocols: in particular, it requires that both the client and server machines have access to the shared secret in cleartext form. CRAM offers a method for avoiding such cleartext storage while retaining the algorithmic simplicity of APOP in using only MD5, though in a "keyed" method. Klensin, Catoe & Krumviede Standards Track [Page 1] RFC 2095 IMAP/POP AUTHorize Extension January 1997 At present, IMAP [IMAP] lacks any facility corresponding to APOP. The only alternative to the strong mechanisms identified in [IMAP- AUTH] is a presumably cleartext username and password, supported through the LOGIN command in [IMAP]. This document describes a simple challenge-response mechanism, similar to APOP and PPP CHAP [PPP], that can be used with IMAP (and, in principle, with POP3). This mechanism also has the advantage over some possible alternatives of not requiring that the server maintain information about email "logins" on a per-login basis. While mechanisms that do require such per-login history records may offer enhanced security, protocols such as IMAP, which may have several connections between a given client and server open more or less simultaneous, may make their implementation particularly challenging. 2. Challenge-Response Authentication Mechanism (CRAM) The authentication type associated with CRAM is "CRAM-MD5". The data encoded in the first ready response contains an presumptively arbitrary string of random digits, a timestamp, and the fully-qualified primary host name of the server. The syntax of the unencoded form must correspond to that of an RFC 822 'msg-id' [RFC822] as described in [POP3]. The client makes note of the data and then responds with a string consisting of the user name, a space, and a 'digest'. The latter is computed by applying the keyed MD5 algorithm from [KEYED-MD5] where the key is a shared secret and the digested text is the timestamp (including angle-brackets). This shared secret is a string known only to the client and server. The `digest' parameter itself is a 16-octet value which is sent in hexadecimal format, using lower-case ASCII characters. When the server receives this client response, it verifies the digest provided. If the digest is correct, the server should consider the client authenticated and respond appropriately. Keyed MD5 is chosen for this application because of the greater security imparted to authentication of short messages. In addition, the use of the techniques described in [KEYED-MD5] for precomputation of intermediate results make it possible to avoid explicit cleartext storage of the shared secret on the server system by instead storing the intermediate results which are known as "contexts". Klensin, Catoe & Krumviede Standards Track [Page 2] RFC 2095 IMAP/POP AUTHorize Extension January 1997 CRAM does not support a protection mechanism. Example: The examples in this document show the use of the CRAM mechanism with the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of the challenges and responses is part of the IMAP4 AUTHENTICATE command, not part of the CRAM specification itself. S: * OK IMAP4 Server C: A0001 AUTHENTICATE CRAM-MD5 S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw S: A0001 OK CRAM authentication successful In this example, the shared secret is the string 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by calculating MD5((tanstaaftanstaaf XOR opad), MD5((tanstaaftanstaaf XOR ipad), <1896.697170952@postoffice.reston.mci.net>)) where ipad and opad are as defined in the keyed-MD5 Work in Progress [KEYED-MD5] and the string shown in the challenge is the base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The shared secret is null-padded to a length of 64 bytes. If the shared secret is longer than 64 bytes, the MD5 digest of the shared secret is used as a 16 byte input to the keyed MD5 calculation. This produces a digest value (in hexadecimal) of b913a602c7eda7a495b4e6e7334d3890 The user name is then prepended to it, forming tim b913a602c7eda7a495b4e6e7334d3890 Which is then base64 encoded to meet the requirements of the IMAP4 AUTHENTICATE command (or the similar POP3 AUTH command), yielding dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw Klensin, Catoe & Krumviede Standards Track [Page 3] RFC 2095 IMAP/POP AUTHorize Extension January 1997 3. References [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", RFC 1334, October 1992. [IMAP] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", RFC 1731, Carnegie Mellon, December 1994. [KEYED-MD5] Krawczyk, H., "HMAC-MD5: Keyed-MD5 for Message Authentication", Work in Progess. [MD5] Rivest, R., "The MD5 Message Digest Algorithm", RFC 1321, MIT Laboratory for Computer Science, April 1992. [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", STD 53, RFC 1939, Carnegie Mellon, May 1996. [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, Carnegie Mellon, December, 1994. 4. Security Considerations It is conjectured that use of the CRAM authentication mechanism provides origin identification and replay protection for a session. Accordingly, a server that implements both a cleartext password command and this authentication type should not allow both methods of access for a given user. While the saving, on the server, of "contexts" (see section 2) is marginally better than saving the shared secrets in cleartext as is required by CHAP [CHAP] and APOP [POP3], it is not sufficient to protect the secrets if the server itself is compromised. Consequently, servers that store the secrets or contexts must both be protected to a level appropriate to the potential information value in user mailboxes and identities. As the length of the shared secret increases, so does the difficulty of deriving it. While there are now suggestions in the literature that the use of MD5 and keyed MD5 in authentication procedures probably has a limited effective lifetime, the technique is now widely deployed and widely understood. It is believed that this general understanding may assist with the rapid replacement, by CRAM-MD5, of the current uses of permanent cleartext passwords in IMAP. This document has been Klensin, Catoe & Krumviede Standards Track [Page 4] RFC 2095 IMAP/POP AUTHorize Extension January 1997 deliberately written to permit easy upgrading to use SHA (or whatever alternatives emerge) when they are considered to be widely available and adequately safe. Even with the use of CRAM, users are still vulnerable to active attacks. An example of an increasingly common active attack is 'TCP Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. See section 1 above for additional discussion. 5. Acknowledgements This memo borrows ideas and some text liberally from [POP3] and [RFC-1731] and thanks are due the authors of those documents. Ran Atkinson made a number of valuable technical and editorial contributions to the document. 6. Authors' Addresses John C. Klensin MCI Telecommunications 800 Boylston St, 7th floor Boston, MA 02199 USA EMail: klensin@mci.net Phone: +1 617 960 1011 Randy Catoe MCI Telecommunications 2100 Reston Parkway Reston, VA 22091 USA EMail: randy@mci.net Phone: +1 703 715 7366 Paul Krumviede MCI Telecommunications 2100 Reston Parkway Reston, VA 22091 USA EMail: paul@mci.net Phone: +1 703 715 7251 Klensin, Catoe & Krumviede Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc2177.IMAP4_IDLE_command.txt ================================================ Network Working Group B. Leiba Request for Comments: 2177 IBM T.J. Watson Research Center Category: Standards Track June 1997 IMAP4 IDLE command Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Abstract The Internet Message Access Protocol [IMAP4] requires a client to poll the server for changes to the selected mailbox (new mail, deletions). It's often more desirable to have the server transmit updates to the client in real time. This allows a user to see new mail immediately. It also helps some real-time applications based on IMAP, which might otherwise need to poll extremely often (such as every few seconds). (While the spec actually does allow a server to push EXISTS responses aysynchronously, a client can't expect this behaviour and must poll.) This document specifies the syntax of an IDLE command, which will allow a client to tell the server that it's ready to accept such real-time updates. 2. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as described in RFC 2060 [IMAP4]. 3. Specification IDLE Command Arguments: none Responses: continuation data will be requested; the client sends the continuation data "DONE" to end the command Leiba Standards Track [Page 1] RFC 2177 IMAP4 IDLE command June 1997 Result: OK - IDLE completed after client sent "DONE" NO - failure: the server will not allow the IDLE command at this time BAD - command unknown or arguments invalid The IDLE command may be used with any IMAP4 server implementation that returns "IDLE" as one of the supported capabilities to the CAPABILITY command. If the server does not advertise the IDLE capability, the client MUST NOT use the IDLE command and must poll for mailbox updates. In particular, the client MUST continue to be able to accept unsolicited untagged responses to ANY command, as specified in the base IMAP specification. The IDLE command is sent from the client to the server when the client is ready to accept unsolicited mailbox update messages. The server requests a response to the IDLE command using the continuation ("+") response. The IDLE command remains active until the client responds to the continuation, and as long as an IDLE command is active, the server is now free to send untagged EXISTS, EXPUNGE, and other messages at any time. The IDLE command is terminated by the receipt of a "DONE" continuation from the client; such response satisfies the server's continuation request. At that point, the server MAY send any remaining queued untagged responses and then MUST immediately send the tagged response to the IDLE command and prepare to process other commands. As in the base specification, the processing of any new command may cause the sending of unsolicited untagged responses, subject to the ambiguity limitations. The client MUST NOT send a command while the server is waiting for the DONE, since the server will not be able to distinguish a command from a continuation. The server MAY consider a client inactive if it has an IDLE command running, and if such a server has an inactivity timeout it MAY log the client off implicitly at the end of its timeout period. Because of that, clients using IDLE are advised to terminate the IDLE and re-issue it at least every 29 minutes to avoid being logged off. This still allows a client to receive immediate mailbox updates even though it need only "poll" at half hour intervals. Leiba Standards Track [Page 2] RFC 2177 IMAP4 IDLE command June 1997 Example: C: A001 SELECT INBOX S: * FLAGS (Deleted Seen) S: * 3 EXISTS S: * 0 RECENT S: * OK [UIDVALIDITY 1] S: A001 OK SELECT completed C: A002 IDLE S: + idling ...time passes; new mail arrives... S: * 4 EXISTS C: DONE S: A002 OK IDLE terminated ...another client expunges message 2 now... C: A003 FETCH 4 ALL S: * 4 FETCH (...) S: A003 OK FETCH completed C: A004 IDLE S: * 2 EXPUNGE S: * 3 EXISTS S: + idling ...time passes; another client expunges message 3... S: * 3 EXPUNGE S: * 2 EXISTS ...time passes; new mail arrives... S: * 3 EXISTS C: DONE S: A004 OK IDLE terminated C: A005 FETCH 3 ALL S: * 3 FETCH (...) S: A005 OK FETCH completed C: A006 IDLE 4. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. Non-terminals referenced but not defined below are as defined by [IMAP4]. command_auth ::= append / create / delete / examine / list / lsub / rename / select / status / subscribe / unsubscribe / idle ;; Valid only in Authenticated or Selected state idle ::= "IDLE" CRLF "DONE" Leiba Standards Track [Page 3] RFC 2177 IMAP4 IDLE command June 1997 5. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. 6. Security Considerations There are no known security issues with this extension. 7. Author's Address Barry Leiba IBM T.J. Watson Research Center 30 Saw Mill River Road Hawthorne, NY 10532 Email: leiba@watson.ibm.com Leiba Standards Track [Page 4] ================================================ FILE: docs/rfcs/rfc2180.IMAP4_multi-accessed_Mailbox_practice.txt ================================================ Network Working Group M. Gahrns Request for Comments: 2180 Microsoft Category: Informational July 1997 IMAP4 Multi-Accessed Mailbox Practice Status of this Memo This memo provides information for the Internet community. This memo does not specify an Internet standard of any kind. Distribution of this memo is unlimited. 1. Abstract IMAP4[RFC-2060] is rich client/server protocol that allows a client to access and manipulate electronic mail messages on a server. Within the protocol framework, it is possible to have differing results for particular client/server interactions. If a protocol does not allow for this, it is often unduly restrictive. For example, when multiple clients are accessing a mailbox and one attempts to delete the mailbox, an IMAP4 server may choose to implement a solution based upon server architectural constraints or individual preference. With this flexibility comes greater client responsibility. It is not sufficient for a client to be written based upon the behavior of a particular IMAP server. Rather the client must be based upon the behavior allowed by the protocol. By documenting common IMAP4 server practice for the case of simultaneous client access to a mailbox, we hope to ensure the widest amount of inter-operation between IMAP4 clients and servers. The behavior described in this document reflects the practice of some existing servers or behavior that the consensus of the IMAP mailing list has deemed to be reasonable. The behavior described within this document is believed to be [RFC-2060] compliant. However, this document is not meant to define IMAP4 compliance, nor is it an exhaustive list of valid IMAP4 behavior. [RFC-2060] must always be consulted to determine IMAP4 compliance, especially for server behavior not described within this document. Gahrns Informational [Page 1] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 2. Conventions used in this document In examples,"C1:", "C2:" and "C3:" indicate lines sent by 3 different clients (client #1, client #2 and client #3) that are connected to a server. "S1:", "S2:" and "S3:" indicated lines sent by the server to client #1, client #2 and client #3 respectively. A shared mailbox, is a mailbox that can be used by multiple users. A multi-accessed mailbox, is a mailbox that has multiple clients simultaneously accessing it. A client is said to have accessed a mailbox after a successful SELECT or EXAMINE command. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC-2119]. 3. Deletion/Renaming of a multi-accessed mailbox If an external agent or multiple clients are accessing a mailbox, care must be taken when handling the deletion or renaming of the mailbox. Following are some strategies an IMAP server may choose to use when dealing with this situation. 3.1. The server MAY fail the DELETE/RENAME command of a multi-accessed mailbox In some cases, this behavior may not be practical. For example, if a large number of clients are accessing a shared mailbox, the window in which no clients have the mailbox accessed may be small or non- existent, effectively rendering the mailbox undeletable or unrenamable. Example: C1: A001 DELETE FOO S1: A001 NO Mailbox FOO is in use by another user. Gahrns Informational [Page 2] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 3.2. The server MAY allow the DELETE command of a multi-accessed mailbox, but keep the information in the mailbox available for those clients that currently have access to the mailbox. When all clients have finished accessing the mailbox, it is permanently removed. For clients that do not already have access to the mailbox, the 'ghosted' mailbox would not be available. For example, it would not be returned to these clients in a subsequent LIST or LSUB command and would not be a valid mailbox argument to any other IMAP command until the reference count of clients accessing the mailbox reached 0. In some cases, this behavior may not be desirable. For example if someone created a mailbox with offensive or sensitive information, one might prefer to have the mailbox deleted and all access to the information contained within removed immediately, rather than continuing to allow access until the client closes the mailbox. Furthermore, this behavior, may prevent 'recycling' of the same mailbox name until all clients have finished accessing the original mailbox. Example: C1: A001 DELETE FOO S1: A001 OK Mailbox FOO is deleted. C2: B001 STORE 1 +FLAGS (\Seen) S2: * 1 FETCH FLAGS (\Seen) S2: B001 OK STORE completed C3: C001 STATUS FOO (MESSAGES) S3: C001 NO Mailbox does not exist C3: C002 CREATE FOO S3: C002 NO Mailbox FOO is still in use. Try again later. Gahrns Informational [Page 3] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 C2: B002 CLOSE S2: B002 OK CLOSE Completed C3: C003 CREATE FOO S3: C003 OK CREATE Completed 3.3. The server MAY allow the DELETE/RENAME of a multi-accessed mailbox, but disconnect all other clients who have the mailbox accessed by sending a untagged BYE response. A server may often choose to disconnect clients in the DELETE case, but may choose to implement a "friendlier" method for the RENAME case. Example: C1: A002 DELETE FOO S1: A002 OK DELETE completed. S2: * BYE Mailbox FOO has been deleted. 3.4. The server MAY allow the RENAME of a multi-accessed mailbox by simply changing the name attribute on the mailbox. Other clients that have access to the mailbox can continue issuing commands such as FETCH that do not reference the mailbox name. Clients would discover the renaming the next time they referred to the old mailbox name. Some servers MAY choose to include the [NEWNAME] response code in their tagged NO response to a command that contained the old mailbox name, as a hint to the client that the operation can succeed if the command is issued with the new mailbox name. Gahrns Informational [Page 4] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 Example: C1: A001 RENAME FOO BAR S1: A001 OK RENAME completed. C2: B001 FETCH 2:4 (FLAGS) S2: * 2 FETCH . . . S2: * 3 FETCH . . . S2: * 4 FETCH . . . S2: B001 OK FETCH completed C2: B002 APPEND FOO {300} C2: Date: Mon, 7 Feb 1994 21:52:25 0800 (PST) C2: . . . S2: B002 NO [NEWNAME FOO BAR] Mailbox has been renamed 4. Expunging of messages on a multi-accessed mailbox If an external agent or multiple clients are accessing a mailbox, care must be taken when handling the EXPUNGE of messages. Other clients accessing the mailbox may be in the midst of issuing a command that depends upon message sequence numbers. Because an EXPUNGE response can not be sent while responding to a FETCH, STORE or SEARCH command, it is not possible to immediately notify the client of the EXPUNGE. This can result in ambiguity if the client issues a FETCH, STORE or SEARCH operation on a message that has been EXPUNGED. 4.1. Fetching of expunged messages Following are some strategies an IMAP server may choose to use when dealing with a FETCH command on expunged messages. Gahrns Informational [Page 5] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 Consider the following scenario: - Client #1 and Client #2 have mailbox FOO selected. - There are 7 messages in the mailbox. - Messages 4:7 are marked for deletion. - Client #1 issues an EXPUNGE, to expunge messages 4:7 4.1.1. The server MAY allow the EXPUNGE of a multi-accessed mailbox but keep the messages available to satisfy subsequent FETCH commands until it is able to send an EXPUNGE response to each client. In some cases, the behavior of keeping "ghosted" messages may not be desirable. For example if a message contained offensive or sensitive information, one might prefer to instantaneously remove all access to the information, regardless of whether another client is in the midst of accessing it. Example: (Building upon the scenario outlined in 4.1.) C2: B001 FETCH 4:7 RFC822 S2: * 4 FETCH RFC822 . . . (RFC822 info returned) S2: * 5 FETCH RFC822 . . . (RFC822 info returned) S2: * 6 FETCH RFC822 . . . (RFC822 info returned) S2: * 7 FETCH RFC822 . . . (RFC822 info returned) S2: B001 OK FETCH Completed C2: B002 NOOP S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 3 EXISTS S2: B002 OK NOOP Complete C2: B003 FETCH 4:7 RFC822 S2: B003 NO Messages 4:7 are no longer available. Gahrns Informational [Page 6] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 4.1.2 The server MAY allow the EXPUNGE of a multi-accessed mailbox, and on subsequent FETCH commands return FETCH responses only for non-expunged messages and a tagged NO. After receiving a tagged NO FETCH response, the client SHOULD issue a NOOP command so that it will be informed of any pending EXPUNGE responses. The client may then either reissue the failed FETCH command, or by examining the EXPUNGE response from the NOOP and the FETCH response from the FETCH, determine that the FETCH failed because of pending expunges. Example: (Building upon the scenario outlined in 4.1.) C2: B001 FETCH 3:5 ENVELOPE S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) S2: B001 NO Some of the requested messages no longer exist C2: B002 NOOP S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 3 EXISTS S2: B002 OK NOOP Completed. Gahrns Informational [Page 7] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 4.1.3 The server MAY allow the EXPUNGE of a multi-accessed mailbox, and on subsequent FETCH commands return the usual FETCH responses for non-expunged messages, "NIL FETCH Responses" for expunged messages, and a tagged OK response. If all of the messages in the subsequent FETCH command have been expunged, the server SHOULD return only a tagged NO. In this case, the client SHOULD issue a NOOP command so that it will be informed of any pending EXPUNGE responses. The client may then either reissue the failed FETCH command, or by examining the EXPUNGE response from the NOOP, determine that the FETCH failed because of pending expunges. "NIL FETCH responses" are a representation of empty data as appropriate for the FETCH argument specified. Example: * 1 FETCH (ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)) * 1 FETCH (FLAGS ()) * 1 FETCH (INTERNALDATE "00-Jan-0000 00:00:00 +0000") * 1 FETCH (RFC822 "") * 1 FETCH (RFC822.HEADER "") * 1 FETCH (RFC822.TEXT "") * 1 FETCH (RFC822.SIZE 0) * 1 FETCH (BODY ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) * 1 FETCH (BODYSTRUCTURE ("TEXT" "PLAIN" NIL NIL NIL "7BIT" 0 0) * 1 FETCH (BODY[
] "") * 1 FETCH (BODY[
] "") In some cases, a client may not be able to distinguish between "NIL FETCH responses" received because a message was expunged and those received because the data actually was NIL. For example, a * 5 FETCH (FLAGS ()) response could be received if no flags were set on message 5, or because message 5 was expunged. In a case of potential ambiguity, the client SHOULD issue a command such as NOOP to force the sending of the EXPUNGE responses to resolve any ambiguity. Example: (Building upon the scenario outlined in 4.1.) Gahrns Informational [Page 8] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 C2: B002 FETCH 3:5 ENVELOPE S2: * 3 FETCH ENVELOPE . . . (ENVELOPE info returned) S2: * 4 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL) S2: * 5 FETCH ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL) S2: B002 OK FETCH Completed C2: B002 FETCH 4:7 ENVELOPE S2: B002 NO Messages 4:7 have been expunged. 4.1.4 To avoid the situation altogether, the server MAY fail the EXPUNGE of a multi-accessed mailbox In some cases, this behavior may not be practical. For example, if a large number of clients are accessing a shared mailbox, the window in which no clients have the mailbox accessed may be small or non- existent, effectively rendering the message unexpungeable. 4.2. Storing of expunged messages Following are some strategies an IMAP server may choose to use when dealing with a STORE command on expunged messages. 4.2.1 If the ".SILENT" suffix is used, and the STORE completed successfully for all the non-expunged messages, the server SHOULD return a tagged OK. Example: (Building upon the scenario outlined in 4.1.) C2: B001 STORE 1:7 +FLAGS.SILENT (\SEEN) S2: B001 OK Gahrns Informational [Page 9] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 4.2.2. If the ".SILENT" suffix is not used, and only expunged messages are referenced, the server SHOULD return only a tagged NO. Example: (Building upon the scenario outlined in 4.1.) C2: B001 STORE 5:7 +FLAGS (\SEEN) S2: B001 NO Messages have been expunged 4.2.3. If the ".SILENT" suffix is not used, and a mixture of expunged and non-expunged messages are referenced, the server MAY set the flags and return a FETCH response for the non-expunged messages along with a tagged NO. After receiving a tagged NO STORE response, the client SHOULD issue a NOOP command so that it will be informed of any pending EXPUNGE responses. The client may then either reissue the failed STORE command, or by examining the EXPUNGE responses from the NOOP and FETCH responses from the STORE, determine that the STORE failed because of pending expunges. Example: (Building upon the scenario outlined in 4.1.) C2: B001 STORE 1:7 +FLAGS (\SEEN) S2: * FETCH 1 FLAGS (\SEEN) S2: * FETCH 2 FLAGS (\SEEN) S2: * FETCH 3 FLAGS (\SEEN) S2: B001 NO Some of the messages no longer exist. C2: B002 NOOP S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 3 EXISTS S2: B002 OK NOOP Completed. Gahrns Informational [Page 10] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 4.2.4. If the ".SILENT" suffix is not used, and a mixture of expunged and non-expunged messages are referenced, the server MAY return an untagged NO and not set any flags. After receiving a tagged NO STORE response, the client SHOULD issue a NOOP command so that it will be informed of any pending EXPUNGE responses. The client would then re-issue the STORE command after updating its message list per any EXPUNGE response. If a large number of clients are accessing a shared mailbox, the window in which there are no pending expunges may be small or non- existent, effectively disallowing a client from setting the flags on all messages at once. Example: (Building upon the scenario outlined in 4.1.) C2: B001 STORE 1:7 +FLAGS (\SEEN) S2: B001 NO Some of the messages no longer exist. C2: B002 NOOP S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 4 EXPUNGE S2: * 3 EXISTS S2: B002 OK NOOP Completed. C2: B003 STORE 1:3 +FLAGS (\SEEN) S2: * FETCH 1 FLAGS (\SEEN) S2: * FETCH 2 FLAGS (\SEEN) S2: * FETCH 3 FLAGS (\SEEN) S2: B003 OK STORE Completed 4.3. Searching of expunged messages A server MAY simply not return a search response for messages that have been expunged and it has not been able to inform the client about. If a client was expecting a particular message to be returned in a search result, and it was not, the client SHOULD issue a NOOP command to see if the message was expunged by another client. Gahrns Informational [Page 11] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 4.4 Copying of expunged messages COPY is the only IMAP4 sequence number command that is safe to allow an EXPUNGE response on. This is because a client is not permitted to cascade several COPY commands together. A client is required to wait and confirm that the copy worked before issuing another one. 4.4.1 The server MAY disallow the COPY of messages in a multi-access mailbox that contains expunged messages. Pending EXPUNGE response(s) MUST be returned to the COPY command. Example: C: A001 COPY 2,4,6,8 FRED S: * 4 EXPUNGE S: A001 NO COPY rejected, because some of the requested messages were expunged Note: Non of the above messages are copied because if a COPY command is unsuccessful, the server MUST restore the destination mailbox to its state before the COPY attempt. 4.4.2 The server MAY allow the COPY of messages in a multi-access mailbox that contains expunged messages. Pending EXPUNGE response(s) MUST be returned to the COPY command. Messages that are copied are messages corresponding to sequence numbers before any EXPUNGE response. Example: C: A001 COPY 2,4,6,8 FRED S: * 3 EXPUNGE S: A001 OK COPY completed In the above example, the messages that are copied to FRED are messages 2,4,6,8 at the start of the COPY command. These are equivalent to messages 2,3,5,7 at the end of the COPY command. The EXPUNGE response can't take place until after the messages from the COPY command are identified (because of the "no expunge while no commands in progress" rule). Gahrns Informational [Page 12] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 Example: C: A001 COPY 2,4,6,8 FRED S: * 4 EXPUNGE S: A001 OK COPY completed In the above example, message 4 was copied before it was expunged, and MUST appear in the destination mailbox FRED. 5. Security Considerations This document describes behavior of servers that use the IMAP4 protocol, and as such, has the same security considerations as described in [RFC-2060]. In particular, some described server behavior does not allow for the immediate deletion of information when a mailbox is accessed by multiple clients. This may be a consideration when dealing with sensitive information where immediate deletion would be preferred. 6. References [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, Harvard University, March 1997. 7. Acknowledgments This document is the result of discussions on the IMAP4 mailing list and is meant to reflect consensus of this group. In particular, Raymond Cheng, Mark Crispin, Jim Evans, Erik Forsberg, Steve Hole, Mark Keasling, Barry Leiba, Syd Logan, John Mani, Pat Moran, Larry Osterman, Chris Newman, Bart Schaefer, Vladimir Vulovic, and Jack De Winter were active participants in this discussion or made suggestions to this document. Gahrns Informational [Page 13] RFC 2180 IMAP4 Multi-Accessed Mailbox Practice July 1997 8. Author's Address Mike Gahrns Microsoft One Microsoft Way Redmond, WA, 98072 Phone: (206) 936-9833 EMail: mikega@microsoft.com Gahrns Informational [Page 14] ================================================ FILE: docs/rfcs/rfc2192.IMAP_URL_scheme.txt ================================================ Network Working Group C. Newman Request for Comments: 2192 Innosoft Category: Standards Track September 1997 IMAP URL Scheme Status of this memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract IMAP [IMAP4] is a rich protocol for accessing remote message stores. It provides an ideal mechanism for accessing public mailing list archives as well as private and shared message stores. This document defines a URL scheme for referencing objects on an IMAP server. 1. Conventions used in this document The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 2. IMAP scheme The IMAP URL scheme is used to designate IMAP servers, mailboxes, messages, MIME bodies [MIME], and search programs on Internet hosts accessible using the IMAP protocol. The IMAP URL follows the common Internet scheme syntax as defined in RFC 1738 [BASIC-URL] except that clear text passwords are not permitted. If : is omitted, the port defaults to 143. Newman Standards Track [Page 1] RFC 2192 IMAP URL Scheme September 1997 An IMAP URL takes one of the following forms: imap:/// imap:///;TYPE= imap:///[uidvalidity][?] imap:///[uidvalidity][isection] The first form is used to refer to an IMAP server, the second form refers to a list of mailboxes, the third form refers to the contents of a mailbox or a set of messages resulting from a search, and the final form refers to a specific message or message part. Note that the syntax here is informal. The authoritative formal syntax for IMAP URLs is defined in section 11. 3. IMAP User Name and Authentication Mechanism A user name and/or authentication mechanism may be supplied. They are used in the "LOGIN" or "AUTHENTICATE" commands after making the connection to the IMAP server. If no user name or authentication mechanism is supplied, the user name "anonymous" is used with the "LOGIN" command and the password is supplied as the Internet e-mail address of the end user accessing the resource. If the URL doesn't supply a user name, the program interpreting the IMAP URL SHOULD request one from the user if necessary. An authentication mechanism can be expressed by adding ";AUTH=" to the end of the user name. When such an is indicated, the client SHOULD request appropriate credentials from that mechanism and use the "AUTHENTICATE" command instead of the "LOGIN" command. If no user name is specified, one SHOULD be obtained from the mechanism or requested from the user as appropriate. The string ";AUTH=*" indicates that the client SHOULD select an appropriate authentication mechanism. It MAY use any mechanism listed in the CAPABILITY command or use an out of band security service resulting in a PREAUTH connection. If no user name is specified and no appropriate authentication mechanisms are available, the client SHOULD fall back to anonymous login as described above. This allows a URL which grants read-write access to authorized users, and read-only anonymous access to other users. If a user name is included with no authentication mechanism, then ";AUTH=*" is assumed. Newman Standards Track [Page 2] RFC 2192 IMAP URL Scheme September 1997 Since URLs can easily come from untrusted sources, care must be taken when resolving a URL which requires or requests any sort of authentication. If authentication credentials are supplied to the wrong server, it may compromise the security of the user's account. The program resolving the URL should make sure it meets at least one of the following criteria in this case: (1) The URL comes from a trusted source, such as a referral server which the client has validated and trusts according to site policy. Note that user entry of the URL may or may not count as a trusted source, depending on the experience level of the user and site policy. (2) Explicit local site policy permits the client to connect to the server in the URL. For example, if the client knows the site domain name, site policy may dictate that any hostname ending in that domain is trusted. (3) The user confirms that connecting to that domain name with the specified credentials and/or mechanism is permitted. (4) A mechanism is used which validates the server before passing potentially compromising client credentials. (5) An authentication mechanism is used which will not reveal information to the server which could be used to compromise future connections. URLs which do not include a user name must be treated with extra care, since they are more likely to compromise the user's primary account. A URL containing ";AUTH=*" must also be treated with extra care since it might fall back on a weaker security mechanism. Finally, clients are discouraged from using a plain text password as a fallback with ";AUTH=*" unless the connection has strong encryption (e.g. a key length of greater than 56 bits). A program interpreting IMAP URLs MAY cache open connections to an IMAP server for later re-use. If a URL contains a user name, only connections authenticated as that user may be re-used. If a URL does not contain a user name or authentication mechanism, then only an anonymous connection may be re-used. If a URL contains an authentication mechanism without a user name, then any non- anonymous connection may be re-used. Note that if unsafe or reserved characters such as " " or ";" are present in the user name or authentication mechanism, they MUST be encoded as described in RFC 1738 [BASIC-URL]. Newman Standards Track [Page 3] RFC 2192 IMAP URL Scheme September 1997 4. IMAP server An IMAP URL referring to an IMAP server has the following form: imap:/// A program interpreting this URL would issue the standard set of commands it uses to present a view of the contents of an IMAP server. This is likely to be semanticly equivalent to one of the following URLs: imap:///;TYPE=LIST imap:///;TYPE=LSUB The program interpreting this URL SHOULD use the LSUB form if it supports mailbox subscriptions. 5. Lists of mailboxes An IMAP URL referring to a list of mailboxes has the following form: imap:///;TYPE= The may be either "LIST" or "LSUB", and is case insensitive. The field ";TYPE=" MUST be included. The is any argument suitable for the list_mailbox field of the IMAP [IMAP4] LIST or LSUB commands. The field may be omitted, in which case the program interpreting the IMAP URL may use "*" or "%" as the . The program SHOULD use "%" if it supports a hierarchical view, otherwise it SHOULD use "*". Note that if unsafe or reserved characters such as " " or "%" are present in they MUST be encoded as described in RFC 1738 [BASIC-URL]. If the character "/" is present in enc_list_mailbox, it SHOULD NOT be encoded. 6. Lists of messages An IMAP URL referring to a list of messages has the following form: imap:///[uidvalidity][?] Newman Standards Track [Page 4] RFC 2192 IMAP URL Scheme September 1997 The field is used as the argument to the IMAP4 "SELECT" command. Note that if unsafe or reserved characters such as " ", ";", or "?" are present in they MUST be encoded as described in RFC 1738 [BASIC-URL]. If the character "/" is present in enc_mailbox, it SHOULD NOT be encoded. The [uidvalidity] field is optional. If it is present, it MUST be the argument to the IMAP4 UIDVALIDITY status response at the time the URL was created. This SHOULD be used by the program interpreting the IMAP URL to determine if the URL is stale. The [?] field is optional. If it is not present, the contents of the mailbox SHOULD be presented by the program interpreting the URL. If it is present, it SHOULD be used as the arguments following an IMAP4 SEARCH command with unsafe characters such as " " (which are likely to be present in the ) encoded as described in RFC 1738 [BASIC-URL]. 7. A specific message or message part An IMAP URL referring to a specific message or message part has the following form: imap:///[uidvalidity][isection] The and [uidvalidity] are as defined above. If [uidvalidity] is present in this form, it SHOULD be used by the program interpreting the URL to determine if the URL is stale. The refers to an IMAP4 message UID, and SHOULD be used as the argument to the IMAP4 "UID FETCH" command. The [isection] field is optional. If not present, the URL refers to the entire Internet message as returned by the IMAP command "UID FETCH BODY.PEEK[]". If present, the URL refers to the object returned by a "UID FETCH BODY.PEEK[
]" command. The type of the object may be determined with a "UID FETCH BODYSTRUCTURE" command and locating the appropriate part in the resulting BODYSTRUCTURE. Note that unsafe characters in [isection] MUST be encoded as described in [BASIC-URL]. Newman Standards Track [Page 5] RFC 2192 IMAP URL Scheme September 1997 8. Relative IMAP URLs Relative IMAP URLs are permitted and are resolved according to the rules defined in RFC 1808 [REL-URL] with one exception. In IMAP URLs, parameters are treated as part of the normal path with respect to relative URL resolution. This is believed to be the behavior of the installed base and is likely to be documented in a future revision of the relative URL specification. The following observations are also important: The grammar element is considered part of the user name for purposes of resolving relative IMAP URLs. This means that unless a new login/server specification is included in the relative URL, the authentication mechanism is inherited from a base IMAP URL. URLs always use "/" as the hierarchy delimiter for the purpose of resolving paths in relative URLs. IMAP4 permits the use of any hierarchy delimiter in mailbox names. For this reason, relative mailbox paths will only work if the mailbox uses "/" as the hierarchy delimiter. Relative URLs may be used on mailboxes which use other delimiters, but in that case, the entire mailbox name MUST be specified in the relative URL or inherited as a whole from the base URL. The base URL for a list of mailboxes or messages which was referred to by an IMAP URL is always the referring IMAP URL itself. The base URL for a message or message part which was referred to by an IMAP URL may be more complicated to determine. The program interpreting the relative URL will have to check the headers of the MIME entity and any enclosing MIME entities in order to locate the "Content-Base" and "Content-Location" headers. These headers are used to determine the base URL as defined in [HTTP]. For example, if the referring IMAP URL contains a "/;SECTION=1.2" parameter, then the MIME headers for section 1.2, for section 1, and for the enclosing message itself SHOULD be checked in that order for "Content-Base" or "Content-Location" headers. 9. Multinational Considerations IMAP4 [IMAP4] section 5.1.3 includes a convention for encoding non-US-ASCII characters in IMAP mailbox names. Because this convention is private to IMAP, it is necessary to convert IMAP's encoding to one that can be more easily interpreted by a URL display program. For this reason, IMAP's modified UTF-7 encoding for mailboxes MUST be converted to UTF-8 [UTF8]. Since 8-bit characters are not permitted in URLs, the UTF-8 characters are Newman Standards Track [Page 6] RFC 2192 IMAP URL Scheme September 1997 encoded as required by the URL specification [BASIC-URL]. Sample code is included in Appendix A to demonstrate this conversion. 10. Examples The following examples demonstrate how an IMAP4 client program might translate various IMAP4 URLs into a series of IMAP4 commands. Commands sent from the client to the server are prefixed with "C:", and responses sent from the server to the client are prefixed with "S:". The URL: Results in the following client commands: C: A001 LOGIN ANONYMOUS sheridan@babylon5.org C: A002 SELECT gray-council C: A003 UID FETCH 20 BODY.PEEK[] The URL: Results in the following client commands: C: A001 LOGIN MICHAEL zipper C: A002 LIST "" users.* The URL: Results in the following client commands: C: A001 LOGIN ANONYMOUS bester@psycop.psicorp.org C: A002 SELECT ~peter/&ZeVnLIqe-/&U,BTFw- Newman Standards Track [Page 7] RFC 2192 IMAP URL Scheme September 1997 The URL: Results in the following client commands: C: A001 AUTHENTICATE KERBEROS_V4 C: A002 SELECT gray-council C: A003 UID FETCH 20 BODY.PEEK[1.2] If the following relative URL is located in that body part: <;section=1.4> This could result in the following client commands: C: A004 UID FETCH 20 (BODY.PEEK[1.2.MIME] BODY.PEEK[1.MIME] BODY.PEEK[HEADER.FIELDS (Content-Base Content-Location)]) C: A005 UID FETCH 20 BODY.PEEK[1.4] The URL: Could result in the following: C: A001 CAPABILITY S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI S: A001 OK C: A002 AUTHENTICATE GSSAPI S: A002 OK user lennier authenticated C: A003 SELECT "gray council" ... C: A004 SEARCH SUBJECT shadows S: * SEARCH 8 10 13 14 15 16 S: A004 OK SEARCH completed C: A005 FETCH 8,10,13:16 ALL ... Newman Standards Track [Page 8] RFC 2192 IMAP URL Scheme September 1997 NOTE: In this final example, the client has implementation dependent choices. The authentication mechanism could be anything, including PREAUTH. And the final FETCH command could fetch more or less information about the messages, depending on what it wishes to display to the user. 11. Security Considerations Security considerations discussed in the IMAP specification [IMAP4] and the URL specification [BASIC-URL] are relevant. Security considerations related to authenticated URLs are discussed in section 3 of this document. Many email clients store the plain text password for later use after logging into an IMAP server. Such clients MUST NOT use a stored password in response to an IMAP URL without explicit permission from the user to supply that password to the specified host name. 12. ABNF for IMAP URL scheme This uses ABNF as defined in RFC 822 [IMAIL]. Terminals from the BNF for IMAP [IMAP4] and URLs [BASIC-URL] are also used. Strings are not case sensitive and free insertion of linear-white-space is not permitted. achar = uchar / "&" / "=" / "~" ; see [BASIC-URL] for "uchar" definition bchar = achar / ":" / "@" / "/" enc_auth_type = 1*achar ; encoded version of [IMAP-AUTH] "auth_type" enc_list_mailbox = 1*bchar ; encoded version of [IMAP4] "list_mailbox" enc_mailbox = 1*bchar ; encoded version of [IMAP4] "mailbox" enc_search = 1*bchar ; encoded version of search_program below enc_section = 1*bchar ; encoded version of section below Newman Standards Track [Page 9] RFC 2192 IMAP URL Scheme September 1997 enc_user = 1*achar ; encoded version of [IMAP4] "userid" imapurl = "imap://" iserver "/" [ icommand ] iauth = ";AUTH=" ( "*" / enc_auth_type ) icommand = imailboxlist / imessagelist / imessagepart imailboxlist = [enc_list_mailbox] ";TYPE=" list_type imessagelist = enc_mailbox [ "?" enc_search ] [uidvalidity] imessagepart = enc_mailbox [uidvalidity] iuid [isection] isection = "/;SECTION=" enc_section iserver = [iuserauth "@"] hostport ; See [BASIC-URL] for "hostport" definition iuid = "/;UID=" nz_number ; See [IMAP4] for "nz_number" definition iuserauth = enc_user [iauth] / [enc_user] iauth list_type = "LIST" / "LSUB" search_program = ["CHARSET" SPACE astring SPACE] search_key *(SPACE search_key) ; IMAP4 literals may not be used ; See [IMAP4] for "astring" and "search_key" section = section_text / (nz_number *["." nz_number] ["." (section_text / "MIME")]) ; See [IMAP4] for "section_text" and "nz_number" uidvalidity = ";UIDVALIDITY=" nz_number ; See [IMAP4] for "nz_number" definition 13. References [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource Locators (URL)", RFC 1738, CERN, Xerox Corporation, University of Minnesota, December 1994. Newman Standards Track [Page 10] RFC 2192 IMAP URL Scheme September 1997 [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanism", RFC 1731, Carnegie-Mellon University, December 1994. [HTTP] Fielding, Gettys, Mogul, Frystyk, Berners-Lee, "Hypertext Transfer Protocol -- HTTP/1.1", RFC 2068, UC Irvine, DEC, MIT/LCS, January 1997. [IMAIL] Crocker, "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822, University of Delaware, August 1982. [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, Harvard University, March 1997. [MIME] Freed, N., Borenstein, N., "Multipurpose Internet Mail Extensions", RFC 2045, Innosoft, First Virtual, November 1996. [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, UC Irvine, June 1995. [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and ISO 10646", RFC 2044, Alis Technologies, October 1996. 14. Author's Address Chris Newman Innosoft International, Inc. 1050 Lakes Drive West Covina, CA 91790 USA EMail: chris.newman@innosoft.com Newman Standards Track [Page 11] RFC 2192 IMAP URL Scheme September 1997 Appendix A. Sample code Here is sample C source code to convert between URL paths and IMAP mailbox names, taking into account mapping between IMAP's modified UTF-7 [IMAP4] and hex-encoded UTF-8 which is more appropriate for URLs. This code has not been rigorously tested nor does it necessarily behave reasonably with invalid input, but it should serve as a useful example. This code just converts the mailbox portion of the URL and does not deal with parameters, query or server components of the URL. #include #include /* hexadecimal lookup table */ static char hex[] = "0123456789ABCDEF"; /* URL unsafe printable characters */ static char urlunsafe[] = " \"#%&+:;<=>?@[\\]^`{|}"; /* UTF7 modified base64 alphabet */ static char base64chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; #define UNDEFINED 64 /* UTF16 definitions */ #define UTF16MASK 0x03FFUL #define UTF16SHIFT 10 #define UTF16BASE 0x10000UL #define UTF16HIGHSTART 0xD800UL #define UTF16HIGHEND 0xDBFFUL #define UTF16LOSTART 0xDC00UL #define UTF16LOEND 0xDFFFUL /* Convert an IMAP mailbox to a URL path * dst needs to have roughly 4 times the storage space of src * Hex encoding can triple the size of the input * UTF-7 can be slightly denser than UTF-8 * (worst case: 8 octets UTF-7 becomes 9 octets UTF-8) */ void MailboxToURL(char *dst, char *src) { unsigned char c, i, bitcount; unsigned long ucs4, utf16, bitbuf; unsigned char base64[256], utf8[6]; Newman Standards Track [Page 12] RFC 2192 IMAP URL Scheme September 1997 /* initialize modified base64 decoding table */ memset(base64, UNDEFINED, sizeof (base64)); for (i = 0; i < sizeof (base64chars); ++i) { base64[base64chars[i]] = i; } /* loop until end of string */ while (*src != '\0') { c = *src++; /* deal with literal characters and &- */ if (c != '&' || *src == '-') { if (c < ' ' || c > '~' || strchr(urlunsafe, c) != NULL) { /* hex encode if necessary */ dst[0] = '%'; dst[1] = hex[c >> 4]; dst[2] = hex[c & 0x0f]; dst += 3; } else { /* encode literally */ *dst++ = c; } /* skip over the '-' if this is an &- sequence */ if (c == '&') ++src; } else { /* convert modified UTF-7 -> UTF-16 -> UCS-4 -> UTF-8 -> HEX */ bitbuf = 0; bitcount = 0; ucs4 = 0; while ((c = base64[(unsigned char) *src]) != UNDEFINED) { ++src; bitbuf = (bitbuf << 6) | c; bitcount += 6; /* enough bits for a UTF-16 character? */ if (bitcount >= 16) { bitcount -= 16; utf16 = (bitcount ? bitbuf >> bitcount : bitbuf) & 0xffff; /* convert UTF16 to UCS4 */ if (utf16 >= UTF16HIGHSTART && utf16 <= UTF16HIGHEND) { ucs4 = (utf16 - UTF16HIGHSTART) << UTF16SHIFT; continue; } else if (utf16 >= UTF16LOSTART && utf16 <= UTF16LOEND) { ucs4 += utf16 - UTF16LOSTART + UTF16BASE; } else { ucs4 = utf16; } Newman Standards Track [Page 13] RFC 2192 IMAP URL Scheme September 1997 /* convert UTF-16 range of UCS4 to UTF-8 */ if (ucs4 <= 0x7fUL) { utf8[0] = ucs4; i = 1; } else if (ucs4 <= 0x7ffUL) { utf8[0] = 0xc0 | (ucs4 >> 6); utf8[1] = 0x80 | (ucs4 & 0x3f); i = 2; } else if (ucs4 <= 0xffffUL) { utf8[0] = 0xe0 | (ucs4 >> 12); utf8[1] = 0x80 | ((ucs4 >> 6) & 0x3f); utf8[2] = 0x80 | (ucs4 & 0x3f); i = 3; } else { utf8[0] = 0xf0 | (ucs4 >> 18); utf8[1] = 0x80 | ((ucs4 >> 12) & 0x3f); utf8[2] = 0x80 | ((ucs4 >> 6) & 0x3f); utf8[3] = 0x80 | (ucs4 & 0x3f); i = 4; } /* convert utf8 to hex */ for (c = 0; c < i; ++c) { dst[0] = '%'; dst[1] = hex[utf8[c] >> 4]; dst[2] = hex[utf8[c] & 0x0f]; dst += 3; } } } /* skip over trailing '-' in modified UTF-7 encoding */ if (*src == '-') ++src; } } /* terminate destination string */ *dst = '\0'; } /* Convert hex coded UTF-8 URL path to modified UTF-7 IMAP mailbox * dst should be about twice the length of src to deal with non-hex * coded URLs */ void URLtoMailbox(char *dst, char *src) { unsigned int utf8pos, utf8total, i, c, utf7mode, bitstogo, utf16flag; unsigned long ucs4, bitbuf; unsigned char hextab[256]; /* initialize hex lookup table */ Newman Standards Track [Page 14] RFC 2192 IMAP URL Scheme September 1997 memset(hextab, 0, sizeof (hextab)); for (i = 0; i < sizeof (hex); ++i) { hextab[hex[i]] = i; if (isupper(hex[i])) hextab[tolower(hex[i])] = i; } utf7mode = 0; utf8total = 0; bitstogo = 0; while ((c = *src) != '\0') { ++src; /* undo hex-encoding */ if (c == '%' && src[0] != '\0' && src[1] != '\0') { c = (hextab[src[0]] << 4) | hextab[src[1]]; src += 2; } /* normal character? */ if (c >= ' ' && c <= '~') { /* switch out of UTF-7 mode */ if (utf7mode) { if (bitstogo) { *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; } *dst++ = '-'; utf7mode = 0; } *dst++ = c; /* encode '&' as '&-' */ if (c == '&') { *dst++ = '-'; } continue; } /* switch to UTF-7 mode */ if (!utf7mode) { *dst++ = '&'; utf7mode = 1; } /* Encode US-ASCII characters as themselves */ if (c < 0x80) { ucs4 = c; utf8total = 1; } else if (utf8total) { /* save UTF8 bits into UCS4 */ ucs4 = (ucs4 << 6) | (c & 0x3FUL); if (++utf8pos < utf8total) { continue; } Newman Standards Track [Page 15] RFC 2192 IMAP URL Scheme September 1997 } else { utf8pos = 1; if (c < 0xE0) { utf8total = 2; ucs4 = c & 0x1F; } else if (c < 0xF0) { utf8total = 3; ucs4 = c & 0x0F; } else { /* NOTE: can't convert UTF8 sequences longer than 4 */ utf8total = 4; ucs4 = c & 0x03; } continue; } /* loop to split ucs4 into two utf16 chars if necessary */ utf8total = 0; do { if (ucs4 >= UTF16BASE) { ucs4 -= UTF16BASE; bitbuf = (bitbuf << 16) | ((ucs4 >> UTF16SHIFT) + UTF16HIGHSTART); ucs4 = (ucs4 & UTF16MASK) + UTF16LOSTART; utf16flag = 1; } else { bitbuf = (bitbuf << 16) | ucs4; utf16flag = 0; } bitstogo += 16; /* spew out base64 */ while (bitstogo >= 6) { bitstogo -= 6; *dst++ = base64chars[(bitstogo ? (bitbuf >> bitstogo) : bitbuf) & 0x3F]; } } while (utf16flag); } /* if in UTF-7 mode, finish in ASCII */ if (utf7mode) { if (bitstogo) { *dst++ = base64chars[(bitbuf << (6 - bitstogo)) & 0x3F]; } *dst++ = '-'; } /* tie off string */ *dst = '\0'; } Newman Standards Track [Page 16] ================================================ FILE: docs/rfcs/rfc2193.IMAP4_Mailbox_referrals.txt ================================================ Network Working Group M. Gahrns Request for Comments: 2193 Microsoft Category: Standards Track September 1997 IMAP4 Mailbox Referrals Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. 1. Abstract When dealing with large amounts of users, messages and geographically dispersed IMAP4 [RFC-2060] servers, it is often desirable to distribute messages amongst different servers within an organization. For example an administrator may choose to store user's personal mailboxes on a local IMAP4 server, while storing shared mailboxes remotely on another server. This type of configuration is common when it is uneconomical to store all data centrally due to limited bandwidth or disk resources. Mailbox referrals allow clients to seamlessly access mailboxes that are distributed across several IMAP4 servers. A referral mechanism can provide efficiencies over the alternative "proxy method", in which the local IMAP4 server contacts the remote server on behalf of the client, and then transfers the data from the remote server to itself, and then on to the client. The referral mechanism's direct client connection to the remote server is often a more efficient use of bandwidth, and does not require the local server to impersonate the client when authenticating to the remote server. 2. Conventions used in this document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. A home server, is an IMAP4 server that contains the user's inbox. A remote mailbox is a mailbox that is not hosted on the user's home server. Gahrns Standards Track [Page 1] RFC 2193 IMAP4 Mailbox Referrals September 1997 A remote server is a server that contains remote mailboxes. A shared mailbox, is a mailbox that multiple users have access to. An IMAP mailbox referral is when the server directs the client to another IMAP mailbox. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC-2119]. 3. Introduction and Overview IMAP4 servers that support this extension MUST list the keyword MAILBOX-REFERRALS in their CAPABILITY response. No client action is needed to invoke the MAILBOX-REFERRALS capability in a server. A MAILBOX-REFERRALS capable IMAP4 server MUST NOT return referrals that result in a referrals loop. A referral response consists of a tagged NO response and a REFERRAL response code. The REFERRAL response code MUST contain as an argument a one or more valid URLs separated by a space as defined in [RFC-1738]. If a server replies with multiple URLs for a particular object, they MUST all be of the same type. In this case, the URL MUST be an IMAP URL as defined in [RFC-2192]. A client that supports the REFERRALS extension MUST be prepared for a URL of any type, but it need only be able to process IMAP URLs. A server MAY respond with multiple IMAP mailbox referrals if there is more than one replica of the mailbox. This allows the implementation of a load balancing or failover scheme. How a server keeps multiple replicas of a mailbox in sync is not addressed by this document. If the server has a preferred order in which the client should attempt to access the URLs, the preferred URL SHOULD be listed in the first, with the remaining URLs presented in descending order of preference. If multiple referrals are given for a mailbox, a server should be aware that there are synchronization issues for a client if the UIDVALIDITY of the referred mailboxes are different. An IMAP mailbox referral may be given in response to an IMAP command that specifies a mailbox as an argument. Gahrns Standards Track [Page 2] RFC 2193 IMAP4 Mailbox Referrals September 1997 Example: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE]Remote Mailbox NOTE: user;AUTH=* is specified as required by [RFC-2192] to avoid a client falling back to anonymous login. Remote mailboxes and their inferiors, that are accessible only via referrals SHOULD NOT appear in LIST and LSUB responses issued against the user's home server. They MUST appear in RLIST and RLSUB responses issued against the user's home server. Hierarchy referrals, in which a client would be required to connect to the remote server to issue a LIST to discover the inferiors of a mailbox are not addressed in this document. For example, if shared mailboxes were only accessible via referrals on a remote server, a RLIST "" "#SHARED/%" command would return the same response if issued against the user's home server or the remote server. Note: Mailboxes that are available on the user's home server do not need to be available on the remote server. In addition, there may be additional mailboxes available on the remote server, but they will not accessible to the client via referrals unless they appear in the LIST response to the RLIST command against the user's home server. A MAILBOX-REFERRALS capable client will issue the RLIST and RLSUB commands in lieu of LIST and LSUB. The RLIST and RLSUB commands behave identically to their LIST and LSUB counterparts, except remote mailboxes are returned in addition to local mailboxes in the LIST and LSUB responses. This avoids displaying to a non MAILBOX-REFERRALS enabled client inaccessible remote mailboxes. 4.1. SELECT, EXAMINE, DELETE, SUBSCRIBE, UNSUBSCRIBE, STATUS and APPEND Referrals An IMAP4 server MAY respond to the SELECT, EXAMINE, DELETE, SUBSCRIBE, UNSUBSCRIBE, STATUS or APPEND command with one or more IMAP mailbox referrals to indicate to the client that the mailbox is hosted on a remote server. When a client processes an IMAP mailbox referral, it will open a new connection or use an existing connection to the remote server so that it is able to issue the commands necessary to process the remote mailbox. Gahrns Standards Track [Page 3] RFC 2193 IMAP4 Mailbox Referrals September 1997 Example: C: A001 DELETE "SHARED/FOO" S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] Remote mailbox. Try SERVER2. S: * OK IMAP4rev1 server ready C: B001 AUTHENTICATE KERBEROS_V4 S: B001 OK user is authenticated C: B002 DELETE "SHARED/FOO" S: B002 OK DELETE completed Example: C: A001 SELECT REMOTE S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/REMOTE IMAP://user;AUTH=*@SERVER3/REMOTE] Remote mailbox. Try SERVER2 or SERVER3. S: * OK IMAP4rev1 server ready C: B001 AUTHENTICATE KERBEROS_V4 S: B001 OK user is authenticated C: B002 SELECT REMOTE S: * 12 EXISTS S: * 1 RECENT S: * OK [UNSEEN 10] Message 10 is first unseen S: * OK [UIDVALIDITY 123456789] S: * FLAGS (Answered Flagged Deleted Seen Draft) S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] S: B002 OK [READ-WRITE] Selected completed C: B003 FETCH 10:12 RFC822 S: * 10 FETCH . . . S: * 11 FETCH . . . S: * 12 FETCH . . . S: B003 OK FETCH Completed Gahrns Standards Track [Page 4] RFC 2193 IMAP4 Mailbox Referrals September 1997 C: B004 LOGOUT S: * BYE IMAP4rev1 server logging out S: B004 OK LOGOUT Completed C: A002 SELECT INBOX S: * 16 EXISTS S: * 2 RECENT S: * OK [UNSEEN 10] Message 10 is first unseen S: * OK [UIDVALIDITY 123456789] S: * FLAGS (Answered Flagged Deleted Seen Draft) S: * OK [PERMANENTFLAGS (Answered Deleted Seen ] S: A002 OK [READ-WRITE] Selected completed 4.2. CREATE Referrals An IMAP4 server MAY respond to the CREATE command with one or more IMAP mailbox referrals, if it wishes to direct the client to issue the CREATE against another server. The server can employ any means, such as examining the hierarchy of the specified mailbox name, in determining which server the mailbox should be created on. Example: C: A001 CREATE "SHARED/FOO" S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] Mailbox should be created on remote server Alternatively, because a home server is required to maintain a listing of referred remote mailboxes, a server MAY allow the creation of a mailbox that will ultimately reside on a remote server against the home server, and provide referrals on subsequent commands that manipulate the mailbox. Example: C: A001 CREATE "SHARED/FOO" S: A001 OK CREATE succeeded C: A002 SELECT "SHARED/FOO" S: A002 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/FOO] Remote mailbox. Try SERVER2 Gahrns Standards Track [Page 5] RFC 2193 IMAP4 Mailbox Referrals September 1997 4.3. RENAME Referrals An IMAP4 server MAY respond to the RENAME command with one or more pairs of IMAP mailbox referrals. In each pair of IMAP mailbox referrals, the first one is an URL to the existing mailbox name and the second is an URL to the requested new mailbox name. If within an IMAP mailbox referral pair, the existing and new mailbox URLs are on different servers, the remote servers are unable to perform the RENAME operation. To achieve the same behavior of server RENAME, the client MAY issue the constituent CREATE, FETCH, APPEND, and DELETE commands against both servers. If within an IMAP mailbox referral pair, the existing and new mailbox URLs are on the same server it is an indication that the currently connected server is unable to perform the operation. The client can simply re-issue the RENAME command on the remote server. Example: C: A001 RENAME FOO BAR S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER1/FOO IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox across servers Since the existing and new mailbox names are on different servers, the client would be required to make a connection to both servers and issue the constituent commands require to achieve the RENAME. Example: C: A001 RENAME FOO BAR S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/FOO IMAP://user;AUTH=*@SERVER2/BAR] Unable to rename mailbox located on SERVER2 Since both the existing and new mailbox are on the same remote server, the client can simply make a connection to the remote server and re-issue the RENAME command. 4.4. COPY Referrals An IMAP4 server MAY respond to the COPY command with one or more IMAP mailbox referrals. This indicates that the destination mailbox is on a remote server. To achieve the same behavior of a server COPY, the client MAY issue the constituent FETCH and APPEND commands against both servers. Gahrns Standards Track [Page 6] RFC 2193 IMAP4 Mailbox Referrals September 1997 Example: C: A001 COPY 1 "SHARED/STUFF" S: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/SHARED/STUFF] Unable to copy message(s) to SERVER2. 5.1 RLIST command Arguments: reference name mailbox name with possible wildcards Responses: untagged responses: LIST Result: OK - RLIST Completed NO - RLIST Failure BAD - command unknown or arguments invalid The RLIST command behaves identically to its LIST counterpart, except remote mailboxes are returned in addition to local mailboxes in the LIST responses. 5.2 RLSUB Command Arguments: reference name mailbox name with possible wildcards Responses: untagged responses: LSUB Result: OK - RLSUB Completed NO - RLSUB Failure BAD - command unknown or arguments invalid The RLSUB command behaves identically to its LSUB counterpart, except remote mailboxes are returned in addition to local mailboxes in the LSUB responses. 6. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. list_mailbox = as defined in [RFC-2060] mailbox = as defined in [RFC-2060] mailbox_referral = SPACE "NO" SPACE (text / text_mime2) ; See [RFC-2060] for , text and text_mime2 definition Gahrns Standards Track [Page 7] RFC 2193 IMAP4 Mailbox Referrals September 1997 referral_response_code = "[" "REFERRAL" 1*(SPACE ) "]" ; See [RFC-1738] for definition rlist = "RLIST" SPACE mailbox SPACE list_mailbox rlsub = "RLSUB" SPACE mailbox SPACE list_mailbox 6. Security Considerations The IMAP4 referral mechanism makes use of IMAP URLs, and as such, have the same security considerations as general internet URLs [RFC- 1738], and in particular IMAP URLs [RFC-2192]. With the MAILBOX-REFERRALS capability, it is potentially easier to write a rogue server that injects a bogus referral response that directs a user to an incorrect mailbox. Although referrals reduce the effort to write such a server, the referral response makes detection of the intrusion easier. 7. References [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [RFC-2192], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, September 1997. [RFC-1738], Berners-Lee, T., Masinter, L., and M. McCahill, "Uniform Resource Locators (URL)", RFC 1738, CERN, Xerox Corporation, University of Minnesota, December 1994. [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, Harvard University, March 1997. [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for Syntax Specifications: ABNF", Work in Progress, Internet Mail Consortium, April 1997. 8. Acknowledgments Many valuable suggestions were received from private discussions and the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, Mark Keasling, Chris Newman and Larry Osterman made significant contributions to this document. Gahrns Standards Track [Page 8] RFC 2193 IMAP4 Mailbox Referrals September 1997 9. Author's Address Mike Gahrns Microsoft One Microsoft Way Redmond, WA, 98072 Phone: (206) 936-9833 EMail: mikega@microsoft.com Gahrns Standards Track [Page 9] ================================================ FILE: docs/rfcs/rfc2195.IMAP-POP_AUTHorize_extension.txt ================================================ Network Working Group J. Klensin Request for Comments: 2195 R. Catoe Category: Standards Track P. Krumviede Obsoletes: 2095 MCI September 1997 IMAP/POP AUTHorize Extension for Simple Challenge/Response Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract While IMAP4 supports a number of strong authentication mechanisms as described in RFC 1731, it lacks any mechanism that neither passes cleartext, reusable passwords across the network nor requires either a significant security infrastructure or that the mail server update a mail-system-wide user authentication file on each mail access. This specification provides a simple challenge-response authentication protocol that is suitable for use with IMAP4. Since it utilizes Keyed-MD5 digests and does not require that the secret be stored in the clear on the server, it may also constitute an improvement on APOP for POP3 use as specified in RFC 1734. 1. Introduction Existing Proposed Standards specify an AUTHENTICATE mechanism for the IMAP4 protocol [IMAP, IMAP-AUTH] and a parallel AUTH mechanism for the POP3 protocol [POP3-AUTH]. The AUTHENTICATE mechanism is intended to be extensible; the four methods specified in [IMAP-AUTH] are all fairly powerful and require some security infrastructure to support. The base POP3 specification [POP3] also contains a lightweight challenge-response mechanism called APOP. APOP is associated with most of the risks associated with such protocols: in particular, it requires that both the client and server machines have access to the shared secret in cleartext form. CRAM offers a method for avoiding such cleartext storage while retaining the algorithmic simplicity of APOP in using only MD5, though in a "keyed" method. Klensin, Catoe & Krumviede Standards Track [Page 1] RFC 2195 IMAP/POP AUTHorize Extension September 1997 At present, IMAP [IMAP] lacks any facility corresponding to APOP. The only alternative to the strong mechanisms identified in [IMAP- AUTH] is a presumably cleartext username and password, supported through the LOGIN command in [IMAP]. This document describes a simple challenge-response mechanism, similar to APOP and PPP CHAP [PPP], that can be used with IMAP (and, in principle, with POP3). This mechanism also has the advantage over some possible alternatives of not requiring that the server maintain information about email "logins" on a per-login basis. While mechanisms that do require such per-login history records may offer enhanced security, protocols such as IMAP, which may have several connections between a given client and server open more or less simultaneous, may make their implementation particularly challenging. 2. Challenge-Response Authentication Mechanism (CRAM) The authentication type associated with CRAM is "CRAM-MD5". The data encoded in the first ready response contains an presumptively arbitrary string of random digits, a timestamp, and the fully-qualified primary host name of the server. The syntax of the unencoded form must correspond to that of an RFC 822 'msg-id' [RFC822] as described in [POP3]. The client makes note of the data and then responds with a string consisting of the user name, a space, and a 'digest'. The latter is computed by applying the keyed MD5 algorithm from [KEYED-MD5] where the key is a shared secret and the digested text is the timestamp (including angle-brackets). This shared secret is a string known only to the client and server. The `digest' parameter itself is a 16-octet value which is sent in hexadecimal format, using lower-case ASCII characters. When the server receives this client response, it verifies the digest provided. If the digest is correct, the server should consider the client authenticated and respond appropriately. Keyed MD5 is chosen for this application because of the greater security imparted to authentication of short messages. In addition, the use of the techniques described in [KEYED-MD5] for precomputation of intermediate results make it possible to avoid explicit cleartext storage of the shared secret on the server system by instead storing the intermediate results which are known as "contexts". Klensin, Catoe & Krumviede Standards Track [Page 2] RFC 2195 IMAP/POP AUTHorize Extension September 1997 CRAM does not support a protection mechanism. Example: The examples in this document show the use of the CRAM mechanism with the IMAP4 AUTHENTICATE command [IMAP-AUTH]. The base64 encoding of the challenges and responses is part of the IMAP4 AUTHENTICATE command, not part of the CRAM specification itself. S: * OK IMAP4 Server C: A0001 AUTHENTICATE CRAM-MD5 S: + PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2UucmVzdG9uLm1jaS5uZXQ+ C: dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw S: A0001 OK CRAM authentication successful In this example, the shared secret is the string 'tanstaaftanstaaf'. Hence, the Keyed MD5 digest is produced by calculating MD5((tanstaaftanstaaf XOR opad), MD5((tanstaaftanstaaf XOR ipad), <1896.697170952@postoffice.reston.mci.net>)) where ipad and opad are as defined in the keyed-MD5 Work in Progress [KEYED-MD5] and the string shown in the challenge is the base64 encoding of <1896.697170952@postoffice.reston.mci.net>. The shared secret is null-padded to a length of 64 bytes. If the shared secret is longer than 64 bytes, the MD5 digest of the shared secret is used as a 16 byte input to the keyed MD5 calculation. This produces a digest value (in hexadecimal) of b913a602c7eda7a495b4e6e7334d3890 The user name is then prepended to it, forming tim b913a602c7eda7a495b4e6e7334d3890 Which is then base64 encoded to meet the requirements of the IMAP4 AUTHENTICATE command (or the similar POP3 AUTH command), yielding dGltIGI5MTNhNjAyYzdlZGE3YTQ5NWI0ZTZlNzMzNGQzODkw Klensin, Catoe & Krumviede Standards Track [Page 3] RFC 2195 IMAP/POP AUTHorize Extension September 1997 3. References [CHAP] Lloyd, B., and W. Simpson, "PPP Authentication Protocols", RFC 1334, October 1992. [IMAP] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [IMAP-AUTH] Myers, J., "IMAP4 Authentication Mechanisms", RFC 1731, Carnegie Mellon, December 1994. [KEYED-MD5] Krawczyk, Bellare, Canetti, "HMAC: Keyed-Hashing for Message Authentication", RFC 2104, February 1997. [MD5] Rivest, R., "The MD5 Message Digest Algorithm", RFC 1321, MIT Laboratory for Computer Science, April 1992. [POP3] Myers, J., and M. Rose, "Post Office Protocol - Version 3", STD 53, RFC 1939, Carnegie Mellon, May 1996. [POP3-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, Carnegie Mellon, December, 1994. 4. Security Considerations It is conjectured that use of the CRAM authentication mechanism provides origin identification and replay protection for a session. Accordingly, a server that implements both a cleartext password command and this authentication type should not allow both methods of access for a given user. While the saving, on the server, of "contexts" (see section 2) is marginally better than saving the shared secrets in cleartext as is required by CHAP [CHAP] and APOP [POP3], it is not sufficient to protect the secrets if the server itself is compromised. Consequently, servers that store the secrets or contexts must both be protected to a level appropriate to the potential information value in user mailboxes and identities. As the length of the shared secret increases, so does the difficulty of deriving it. While there are now suggestions in the literature that the use of MD5 and keyed MD5 in authentication procedures probably has a limited effective lifetime, the technique is now widely deployed and widely understood. It is believed that this general understanding may assist with the rapid replacement, by CRAM-MD5, of the current uses of permanent cleartext passwords in IMAP. This document has been Klensin, Catoe & Krumviede Standards Track [Page 4] RFC 2195 IMAP/POP AUTHorize Extension September 1997 deliberately written to permit easy upgrading to use SHA (or whatever alternatives emerge) when they are considered to be widely available and adequately safe. Even with the use of CRAM, users are still vulnerable to active attacks. An example of an increasingly common active attack is 'TCP Session Hijacking' as described in CERT Advisory CA-95:01 [CERT95]. See section 1 above for additional discussion. 5. Acknowledgements This memo borrows ideas and some text liberally from [POP3] and [RFC-1731] and thanks are due the authors of those documents. Ran Atkinson made a number of valuable technical and editorial contributions to the document. 6. Authors' Addresses John C. Klensin MCI Telecommunications 800 Boylston St, 7th floor Boston, MA 02199 USA EMail: klensin@mci.net Phone: +1 617 960 1011 Randy Catoe MCI Telecommunications 2100 Reston Parkway Reston, VA 22091 USA EMail: randy@mci.net Phone: +1 703 715 7366 Paul Krumviede MCI Telecommunications 2100 Reston Parkway Reston, VA 22091 USA EMail: paul@mci.net Phone: +1 703 715 7251 Klensin, Catoe & Krumviede Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc2221.IMAP4_Login_referrals.txt ================================================ Network Working Group M. Gahrns Request for Comments: 2221 Microsoft Category: Standards Track October 1997 IMAP4 Login Referrals Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (1997). All Rights Reserved. 1. Abstract When dealing with large amounts of users and many IMAP4 [RFC-2060] servers, it is often necessary to move users from one IMAP4 server to another. For example, hardware failures or organizational changes may dictate such a move. Login referrals allow clients to transparently connect to an alternate IMAP4 server, if their home IMAP4 server has changed. A referral mechanism can provide efficiencies over the alternative 'proxy method', in which the local IMAP4 server contacts the remote server on behalf of the client, and then transfers the data from the remote server to itself, and then on to the client. The referral mechanism's direct client connection to the remote server is often a more efficient use of bandwidth, and does not require the local server to impersonate the client when authenticating to the remote server. 2. Conventions used in this document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. A home server, is an IMAP4 server that contains the user's inbox. A remote server is a server that contains remote mailboxes. Gahrns Standards Track [Page 1] RFC 2221 IMAP4 Login Referrals October 1997 The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC-2119]. 3. Introduction and Overview IMAP4 servers that support this extension MUST list the keyword LOGIN-REFERRALS in their CAPABILITY response. No client action is needed to invoke the LOGIN-REFERRALS capability in a server. A LOGIN-REFERRALS capable IMAP4 server SHOULD NOT return a referral to a server that will return a referral. A client MUST NOT follow more than 10 levels of referral without consulting the user. A LOGIN-REFERRALS response code MUST contain as an argument a valid IMAP server URL as defined in [IMAP-URL]. A home server referral consists of either a tagged NO or OK, or an untagged BYE response that contains a LOGIN-REFERRALS response code. Example: A001 NO [REFERRAL IMAP://user;AUTH=*@SERVER2/] Remote Server NOTE: user;AUTH=* is specified as required by [IMAP-URL] to avoid a client falling back to anonymous login. 4. Home Server Referrals A home server referral may be returned in response to an AUTHENTICATE or LOGIN command, or it may appear in the connection startup banner. If a server returns a home server referral in a tagged NO response, that server does not contain any mailboxes that are accessible to the user. If a server returns a home server referral in a tagged OK response, it indicates that the user's personal mailboxes are elsewhere, but the server contains public mailboxes which are readable by the user. After receiving a home server referral, the client can not make any assumptions as to whether this was a permanent or temporary move of the user. 4.1. LOGIN and AUTHENTICATE Referrals An IMAP4 server MAY respond to a LOGIN or AUTHENTICATE command with a home server referral if it wishes to direct the user to another IMAP4 server. Example: C: A001 LOGIN MIKE PASSWORD S: A001 NO [REFERRAL IMAP://MIKE@SERVER2/] Specified user is invalid on this server. Try SERVER2. Gahrns Standards Track [Page 2] RFC 2221 IMAP4 Login Referrals October 1997 Example: C: A001 LOGIN MATTHEW PASSWORD S: A001 OK [REFERRAL IMAP://MATTHEW@SERVER2/] Specified user's personal mailboxes located on Server2, but public mailboxes are available. Example: C: A001 AUTHENTICATE GSSAPI S: A001 NO [REFERRAL IMAP://user;AUTH=GSSAPI@SERVER2/] Specified user is invalid on this server. Try SERVER2. 4.2. BYE at connection startup referral An IMAP4 server MAY respond with an untagged BYE and a REFERRAL response code that contains an IMAP URL to a home server if it is not willing to accept connections and wishes to direct the client to another IMAP4 server. Example: S: * BYE [REFERRAL IMAP://user;AUTH=*@SERVER2/] Server not accepting connections. Try SERVER2 5. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. This amends the "resp_text_code" element of the IMAP4 grammar described in [RFC-2060] resp_text_code =/ "REFERRAL" SPACE ; See [IMAP-URL] for definition of ; See [RFC-2060] for base definition of resp_text_code 6. Security Considerations The IMAP4 login referral mechanism makes use of IMAP URLs, and as such, have the same security considerations as general internet URLs [RFC-1738], and in particular IMAP URLs [IMAP-URL]. A server MUST NOT give a login referral if authentication for that user fails. This is to avoid revealing information about the user's account to an unauthorized user. With the LOGIN-REFERRALS capability, it is potentially easier to write a rogue 'password catching' server that collects login data and then refers the client to their actual IMAP4 server. Although referrals reduce the effort to write such a server, the referral response makes detection of the intrusion easier. Gahrns Standards Track [Page 3] RFC 2221 IMAP4 Login Referrals October 1997 7. References [RFC-2060], Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, Innosoft, September 1997. [RFC-1738], Berners-Lee, T., Masinter, L. and M. McCahill, "Uniform Resource Locators (URL)", RFC 1738, December 1994. [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, March 1997. [ABNF], DRUMS working group, Dave Crocker Editor, "Augmented BNF for Syntax Specifications: ABNF", Work in Progress. 8. Acknowledgments Many valuable suggestions were received from private discussions and the IMAP4 mailing list. In particular, Raymond Cheng, Mark Crispin, Mark Keasling Chris Newman and Larry Osterman made significant contributions to this document. 9. Author's Address Mike Gahrns Microsoft One Microsoft Way Redmond, WA, 98072 Phone: (206) 936-9833 EMail: mikega@microsoft.com Gahrns Standards Track [Page 4] RFC 2221 IMAP4 Login Referrals October 1997 10. Full Copyright Statement Copyright (C) The Internet Society (1997). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implmentation may be prepared, copied, published andand distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE." Gahrns Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc2244.ACAP.txt ================================================ Network Working Group C. Newman Request for Comments: 2244 Innosoft Category: Standards Track J. G. Myers Netscape November 1997 ACAP -- Application Configuration Access Protocol Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society 1997. All Rights Reserved. Abstract The Application Configuration Access Protocol (ACAP) is designed to support remote storage and access of program option, configuration and preference information. The data store model is designed to allow a client relatively simple access to interesting data, to allow new information to be easily added without server re-configuration, and to promote the use of both standardized data and custom or proprietary data. Key features include "inheritance" which can be used to manage default values for configuration settings and access control lists which allow interesting personal information to be shared and group information to be restricted. Newman & Myers Standards Track [Page i] RFC 2244 ACAP November 1997 Table of Contents Status of this Memo ............................................... i Copyright Notice .................................................. i Abstract .......................................................... i ACAP Protocol Specification ....................................... 1 1. Introduction ............................................. 1 1.1. Conventions Used in this Document ........................ 1 1.2. ACAP Data Model .......................................... 1 1.3. ACAP Design Goals ........................................ 1 1.4. Validation ............................................... 2 1.5. Definitions .............................................. 2 1.6. ACAP Command Overview .................................... 4 2. Protocol Framework ....................................... 4 2.1. Link Level ............................................... 4 2.2. Commands and Responses ................................... 4 2.2.1. Client Protocol Sender and Server Protocol Receiver ...... 4 2.2.2. Server Protocol Sender and Client Protocol Receiver ...... 5 2.3. Server States ............................................ 6 2.3.1. Non-Authenticated State .................................. 6 2.3.2. Authenticated State ...................................... 6 2.3.3. Logout State ............................................. 6 2.4. Operational Considerations ............................... 7 2.4.1. Untagged Status Updates .................................. 7 2.4.2. Response when No Command in Progress ..................... 7 2.4.3. Auto-logout Timer ........................................ 7 2.4.4. Multiple Commands in Progress ............................ 8 2.5. Server Command Continuation Request ...................... 8 2.6. Data Formats ............................................. 8 2.6.1. Atom ..................................................... 9 2.6.2. Number ................................................... 9 2.6.3. String ................................................... 9 2.6.3.1. 8-bit and Binary Strings ................................. 10 2.6.4. Parenthesized List ....................................... 10 2.6.5. NIL ...................................................... 10 3. Protocol Elements ........................................ 10 3.1. Entries and Attributes ................................... 10 3.1.1. Predefined Attributes .................................... 11 3.1.2. Attribute Metadata ....................................... 12 3.2. ACAP URL scheme .......................................... 13 3.2.1. ACAP URL User Name and Authentication Mechanism .......... 13 3.2.2. Relative ACAP URLs ....................................... 14 3.3. Contexts ................................................. 14 Newman & Myers Standards Track [Page ii] RFC 2244 ACAP November 1997 3.4. Comparators .............................................. 15 3.5. Access Control Lists (ACLs) .............................. 17 3.6. Server Response Codes .................................... 18 4. Namespace Conventions .................................... 21 4.1. Dataset Namespace ........................................ 21 4.2. Attribute Namespace ...................................... 21 4.3. Formal Syntax for Dataset and Attribute Namespace ........ 22 5. Dataset Management ....................................... 23 5.1. Dataset Inheritance ...................................... 23 5.2. Dataset Attributes ....................................... 24 5.3. Dataset Creation ......................................... 25 5.4. Dataset Class Capabilities ............................... 25 5.5. Dataset Quotas ........................................... 26 6. Command and Response Specifications ...................... 26 6.1. Initial Connection ....................................... 26 6.1.1. ACAP Untagged Response ................................... 26 6.2. Any State ................................................ 27 6.2.1. NOOP Command ............................................. 27 6.2.2. LANG Command ............................................. 28 6.2.3. LANG Intermediate Response ............................... 28 6.2.4. LOGOUT Command ........................................... 29 6.2.5. OK Response .............................................. 29 6.2.6. NO Response .............................................. 29 6.2.7. BAD Response ............................................. 30 6.2.8. BYE Untagged Response .................................... 30 6.2.9. ALERT Untagged Response .................................. 31 6.3. Non-Authenticated State .................................. 31 6.3.1. AUTHENTICATE Command ..................................... 31 6.4. Searching ................................................ 33 6.4.1. SEARCH Command ........................................... 33 6.4.2. ENTRY Intermediate Response .............................. 37 6.4.3. MODTIME Intermediate Response ............................ 38 6.4.4. REFER Intermediate Response .............................. 38 6.4.5. Search Examples .......................................... 38 6.5. Contexts ................................................. 39 6.5.1. FREECONTEXT Command ...................................... 39 6.5.2. UPDATECONTEXT Command .................................... 40 6.5.3. ADDTO Untagged Response .................................. 40 6.5.4. REMOVEFROM Untagged Response ............................. 41 6.5.5. CHANGE Untagged Response ................................. 41 6.5.6. MODTIME Untagged Response ................................ 42 6.6. Dataset modification ..................................... 42 6.6.1. STORE Command ............................................ 42 6.6.2. DELETEDSINCE Command ..................................... 45 6.6.3. DELETED Intermediate Response ............................ 45 6.7. Access Control List Commands ............................. 45 6.7.1. SETACL Command ........................................... 46 6.7.2. DELETEACL Command ........................................ 46 Newman & Myers Standards Track [Page iii] RFC 2244 ACAP November 1997 6.7.3. MYRIGHTS Command ......................................... 47 6.7.4. MYRIGHTS Intermediate Response ........................... 47 6.7.5. LISTRIGHTS Command ....................................... 47 6.7.6. LISTRIGHTS Intermediate Response ......................... 48 6.8. Quotas ................................................... 48 6.8.1. GETQUOTA Command ......................................... 48 6.8.3. QUOTA Untagged Response .................................. 49 6.9. Extensions ............................................... 49 7. Registration Procedures .................................. 49 7.1. ACAP Capabilities ........................................ 50 7.2. ACAP Response Codes ...................................... 50 7.3. Dataset Classes .......................................... 51 7.4. Vendor Subtree ........................................... 51 8. Formal Syntax ............................................ 52 9. Multi-lingual Considerations ............................. 61 10. Security Considerations .................................. 62 11. Acknowledgments .......................................... 63 12. Authors' Addresses ....................................... 63 Appendices ........................................................ 64 A. References ............................................... 64 B. ACAP Keyword Index ....................................... 66 C. Full Copyright Statement Newman & Myers Standards Track [Page iv] RFC 2244 ACAP November 1997 ACAP Protocol Specification 1. Introduction 1.1. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. If such lines are wrapped without a new "C:" or "S:" label, then the wrapping is for editorial clarity and is not part of the command. The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as described in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 1.2. ACAP Data Model An ACAP server exports a hierarchical tree of entries. Each level of the tree is called a dataset, and each dataset is made up of a list of entries. Each entry has a unique name and may contain any number of named attributes. Each attribute within an entry may be single valued or multi-valued and may have associated metadata to assist access and interpretation of the value. The rules with which a client interprets the data within a portion of ACAP's tree of entries are called a dataset class. 1.3. ACAP Design Goals ACAP's primary purpose is to allow users access to their configuration data from multiple network-connected computers. Users can then sit down in front of any network-connected computer, run any ACAP-enabled application and have access to their own configuration data. Because it is hoped that many applications will become ACAP- enabled, client simplicity was preferred to server or protocol simplicity whenever reasonable. ACAP is designed to be easily manageable. For this reason, it includes "inheritance" which allows one dataset to inherit default attributes from another dataset. In addition, access control lists are included to permit delegation of management and quotas are included to control storage. Finally, an ACAP server which is conformant to this base specification should be able to support most dataset classes defined in the future without requiring a server reconfiguration or upgrade. Newman & Myers Standards Track [Page 1] RFC 2244 ACAP November 1997 ACAP is designed to operate well with a client that only has intermittent access to an ACAP server. For this reason, each entry has a server maintained modification time so that the client may detect changes. In addition, the client may ask the server for a list of entries which have been removed since it last accessed the server. ACAP presumes that a dataset may be potentially large and/or the client's network connection may be slow, and thus offers server sorting, selective fetching and change notification for entries within a dataset. As required for most Internet protocols, security, scalability and internationalization were important design goals. Given these design goals, an attempt was made to keep ACAP as simple as possible. It is a traditional Internet text based protocol which massively simplifies protocol debugging. It was designed based on the successful IMAP [IMAP4] protocol framework, with a few refinements. 1.4. Validation By default, any value may be stored in any attribute for which the user has appropriate permission and quota. This rule is necessary to allow the addition of new simple dataset classes without reconfiguring or upgrading the server. In some cases, such as when the value has special meaning to the server, it is useful to have the server enforce validation by returning the INVALID response code to a STORE command. These cases MUST be explicitly identified in the dataset class specification which SHOULD include specific fixed rules for validation. Since a given ACAP server may be unaware of any particular dataset class specification, clients MUST NOT depend on the presence of enforced validation on the server. 1.5. Definitions access control list (ACL) A set of identifier, rights pairs associated with an object. An ACL is used to determine which operations a user is permitted to perform on that object. See section 3.5. attribute A named value within an entry. See section 3.1. Newman & Myers Standards Track [Page 2] RFC 2244 ACAP November 1997 comparator A named function which can be used to perform one or more of three comparison operations: ordering, equality and substring matching. See section 3.4. context An ordered subset of entries in a dataset, created by a SEARCH command with a MAKECONTEXT modifier. See section 3.3. dataset One level of hierarchy in ACAP's tree of entries. dataset class specification The rules which allow a client to interpret the data within a portion of ACAP's tree of entries. entry A set of attributes with a unique entry name. See section 3.1. metadata Information describing an attribute, its value and any access controls associated with that attribute. See section 3.1.2. NIL This represents the non-existence of a particular data item. NUL A control character encoded as 0 in US-ASCII [US-ASCII]. octet An 8-bit value. On most modern computer systems, an octet is one byte. SASL Simple Authentication and Security Layer [SASL]. UTC Universal Coordinated Time as maintained by the Bureau International des Poids et Mesures (BIPM). UTF-8 An 8-bit transformation format of the Universal Character Set [UTF8]. Note that an incompatible change was made to the coded character set referenced by [UTF8], so for the purpose of this document, UTF-8 refers to the UTF-8 encoding as defined by version 2.0 of Unicode [UNICODE-2], or ISO 10646 [ISO-10646] including amendments one through seven. Newman & Myers Standards Track [Page 3] RFC 2244 ACAP November 1997 1.6. ACAP Command Overview The AUTHENTICATE, NOOP, LANG and LOGOUT commands provide basic protocol services. The SEARCH command is used to select, sort, fetch and monitor changes to attribute values and metadata. The UPDATECONTEXT and FREECONTEXT commands are also used to assist in monitoring changes in attribute values and metadata. The STORE command is used to add, modify and delete entries and attributes. The DELETEDSINCE command is used to assist a client in re-synchronizing a cache with the server. The GETQUOTA, SETACL, DELETEACL, LISTRIGHTS and MYRIGHTS commands are used to examine storage quotas and examine or modify access permissions. 2. Protocol Framework 2.1. Link Level The ACAP protocol assumes a reliable data stream such as provided by TCP. When TCP is used, an ACAP server listens on port 674. 2.2. Commands and Responses An ACAP session consists of the establishment of a client/server connection, an initial greeting from the server, and client/server interactions. These client/server interactions consist of a client command, server data, and a server completion result. ACAP is a text-based line-oriented protocol. In general, interactions transmitted by clients and servers are in the form of lines; that is, sequences of characters that end with a CRLF. The protocol receiver of an ACAP client or server is either reading a line, or is reading a sequence of octets with a known count (a literal) followed by a line. Both clients and servers must be capable of handling lines of arbitrary length. 2.2.1. Client Protocol Sender and Server Protocol Receiver The client command begins an operation. Each client command is prefixed with a identifier (an alphanumeric string of no more than 32 characters, e.g., A0001, A0002, etc.) called a "tag". A different tag SHOULD be generated by the client for each command. There are two cases in which a line from the client does not represent a complete command. In one case, a command argument is quoted with an octet count (see the description of literal in section 2.6.3); in the other case, the command arguments require server Newman & Myers Standards Track [Page 4] RFC 2244 ACAP November 1997 feedback (see the AUTHENTICATE command). In some of these cases, the server sends a command continuation request if it is ready for the next part of the command. This response is prefixed with the token "+". Note: If, instead, the server detected an error in a command, it sends a BAD completion response with tag matching the command (as described below) to reject the command and prevent the client from sending any more of the command. It is also possible for the server to send a completion or intermediate response for some other command (if multiple commands are in progress), or untagged data. In either case, the command continuation request is still pending; the client takes the appropriate action for the response, and reads another response from the server. The ACAP server reads a command line from the client, parses the command and its arguments, and transmits server data and a server command completion result. 2.2.2. Server Protocol Sender and Client Protocol Receiver Data transmitted by the server to the client come in four forms: command continuation requests, command completion results, intermediate responses, and untagged responses. A command continuation request is prefixed with the token "+". A command completion result indicates the success or failure of the operation. It is tagged with the same tag as the client command which began the operation. Thus, if more than one command is in progress, the tag in a server completion response identifies the command to which the response applies. There are three possible server completion responses: OK (indicating success), NO (indicating failure), or BAD (indicating protocol error such as unrecognized command or command syntax error). An intermediate response returns data which can only be interpreted within the context of a command in progress. It is tagged with the same tag as the client command which began the operation. Thus, if more than one command is in progress, the tag in an intermediate response identifies the command to which the response applies. A tagged response other than "OK", "NO", or "BAD" is an intermediate response. Newman & Myers Standards Track [Page 5] RFC 2244 ACAP November 1997 An untagged response returns data or status messages which may be interpreted outside the context of a command in progress. It is prefixed with the token "*". Untagged data may be sent as a result of a client command, or may be sent unilaterally by the server. There is no syntactic difference between untagged data that resulted from a specific command and untagged data that were sent unilaterally. The protocol receiver of an ACAP client reads a response line from the server. It then takes action on the response based upon the first token of the response, which may be a tag, a "*", or a "+" as described above. A client MUST be prepared to accept any server response at all times. This includes untagged data that it may not have requested. This topic is discussed in greater detail in the Server Responses section. 2.3. Server States An ACAP server is in one of three states. Most commands are valid in only certain states. It is a protocol error for the client to attempt a command while the server is in an inappropriate state for that command. In this case, a server will respond with a BAD command completion result. 2.3.1. Non-Authenticated State In non-authenticated state, the user must supply authentication credentials before most commands will be permitted. This state is entered when a connection starts. 2.3.2. Authenticated State In authenticated state, the user is authenticated and most commands will be permitted. This state is entered when acceptable authentication credentials have been provided. 2.3.3. Logout State In logout state, the session is being terminated, and the server will close the connection. This state can be entered as a result of a client request or by unilateral server decision. Newman & Myers Standards Track [Page 6] RFC 2244 ACAP November 1997 +--------------------------------------+ |initial connection and server greeting| +--------------------------------------+ || (1) || (2) VV || +-----------------+ || |non-authenticated| || +-----------------+ || || (4) || (3) || || VV || || +----------------+ || || | authenticated | || || +----------------+ || || || (4) || VV VV VV +--------------------------------------+ | logout and close connection | +--------------------------------------+ (1) connection (ACAP greeting) (2) rejected connection (BYE greeting) (3) successful AUTHENTICATE command (4) LOGOUT command, server shutdown, or connection closed 2.4. Operational Considerations 2.4.1. Untagged Status Updates At any time, a server can send data that the client did not request. 2.4.2. Response when No Command in Progress Server implementations are permitted to send an untagged response while there is no command in progress. Server implementations that send such responses MUST deal with flow control considerations. Specifically, they must either (1) verify that the size of the data does not exceed the underlying transport's available window size, or (2) use non-blocking writes. 2.4.3. Auto-logout Timer If a server has an inactivity auto-logout timer, that timer MUST be of at least 30 minutes duration. The receipt of ANY command from the client during that interval MUST suffice to reset the auto-logout timer. Newman & Myers Standards Track [Page 7] RFC 2244 ACAP November 1997 2.4.4. Multiple Commands in Progress The client is not required to wait for the completion result of a command before sending another command, subject to flow control constraints on the underlying data stream. Similarly, a server is not required to process a command to completion before beginning processing of the next command, unless an ambiguity would result because of a command that would affect the results of other commands. If there is such an ambiguity, the server executes commands to completion in the order given by the client. 2.5. Server Command Continuation Request The command continuation request is indicated by a "+" token instead of a tag. This indicates that the server is ready to accept the continuation of a command from the client. This response is used in the AUTHENTICATE command to transmit server data to the client, and request additional client data. This response is also used if an argument to any command is a synchronizing literal (see section 2.6.3). The client is not permitted to send the octets of a synchronizing literal unless the server indicates that it expects it. This permits the server to process commands and reject errors on a line-by-line basis, assuming it checks for non-synchronizing literals at the end of each line. The remainder of the command, including the CRLF that terminates a command, follows the octets of the literal. If there are any additional command arguments the literal octets are followed by a space and those arguments. Example: C: A099 FREECONTEXT {10} S: + "Ready for additional command text" C: FRED C: FOOB S: A099 OK "FREECONTEXT completed" C: A044 BLURDYBLOOP {102856} S: A044 BAD "No such command as 'BLURDYBLOOP'" 2.6. Data Formats ACAP uses textual commands and responses. Data in ACAP can be in one of five forms: atom, number, string, parenthesized list or NIL. Newman & Myers Standards Track [Page 8] RFC 2244 ACAP November 1997 2.6.1. Atom An atom consists of one to 1024 non-special characters. It must begin with a letter. Atoms are used for protocol keywords. 2.6.2. Number A number consists of one or more digit characters, and represents a numeric value. Numbers are restricted to the range of an unsigned 32-bit integer: 0 < number < 4,294,967,296. 2.6.3. String A string is in one of two forms: literal and quoted string. The literal form is the general form of string. The quoted string form is an alternative that avoids the overhead of processing a literal at the cost of restrictions of what may be in a quoted string. A literal is a sequence of zero or more octets (including CR and LF), prefix-quoted with an octet count in the form of an open brace ("{"), the number of octets, close brace ("}"), and CRLF. In the case of literals transmitted from server to client, the CRLF is immediately followed by the octet data. There are two forms of literals transmitted from client to server. The form where the open brace ("{") and number of octets is immediately followed by a close brace ("}") and CRLF is called a synchronizing literal. When sending a synchronizing literal, the client must wait to receive a command continuation request before sending the octet data (and the remainder of the command). The other form of literal, the non-synchronizing literal, is used to transmit a string from client to server without waiting for a command continuation request. The non-synchronizing literal differs from the synchronizing literal by having a plus ("+") between the number of octets and the close brace ("}") and by having the octet data immediately following the CRLF. A quoted string is a sequence of zero to 1024 octets excluding NUL, CR and LF, with double quote (<">) characters at each end. The empty string is represented as "" (a quoted string with zero characters between double quotes), as {0} followed by CRLF (a synchronizing literal with an octet count of 0), or as {0+} followed by a CRLF (a non-synchronizing literal with an octet count of 0). Note: Even if the octet count is 0, a client transmitting a synchronizing literal must wait to receive a command continuation request. Newman & Myers Standards Track [Page 9] RFC 2244 ACAP November 1997 2.6.3.1. 8-bit and Binary Strings Most strings in ACAP are restricted to UTF-8 characters and may not contain NUL octets. Attribute values MAY contain any octets including NUL. 2.6.4. Parenthesized List Data structures are represented as a "parenthesized list"; a sequence of data items, delimited by space, and bounded at each end by parentheses. A parenthesized list can contain other parenthesized lists, using multiple levels of parentheses to indicate nesting. The empty list is represented as () -- a parenthesized list with no members. 2.6.5. NIL The special atom "NIL" represents the non-existence of a particular data item that is represented as a string or parenthesized list, as distinct from the empty string "" or the empty parenthesized list (). 3. Protocol Elements This section defines data formats and other protocol elements used throughout the ACAP protocol. 3.1. Entries and Attributes Within a dataset, each entry name is made up of zero or more UTF-8 characters other than slash ("/"). A slash separated list of entries, one at each level of the hierarchy, forms the full path to an entry. Each entry is made up of a set of attributes. Each attribute has a hierarchical name in UTF-8, with each component of the name separated by a period ("."). The value of an attribute is either single or multi-valued. A single value is NIL (has no value), or a string of zero or more octets. A multi-value is a list of zero or more strings, each of zero or more octets. Attribute names are not permitted to contain asterisk ("*") or percent ("%") and MUST be valid UTF-8 strings which do not contain NUL. Invalid attribute names result in a BAD response. Entry names Newman & Myers Standards Track [Page 10] RFC 2244 ACAP November 1997 are not permitted to begin with "." or contain slash ("/") and MUST be valid UTF-8 strings which do not contain NUL. Invalid entry names in the entry field of a command result in a BAD response. Use of non-visible UTF-8 characters in attribute and entry names is discouraged. 3.1.1. Predefined Attributes Attribute names which do not contain a dot (".") are reserved for standardized attributes which have meaning in any dataset. The following attributes are defined by the ACAP protocol. entry Contains the name of the entry. MUST be single valued. Attempts to use illegal or multi-valued values for the entry attribute are protocol errors and MUST result in a BAD completion response. This is a special case. modtime Contains the date and time any read-write metadata in the entry was last modified. This value MUST be in UTC, MUST be automatically updated by the server. The value consists of 14 or more US-ASCII digits. The first four indicate the year, the next two indicate the month, the next two indicate the day of month, the next two indicate the hour (0 - 23), the next two indicate the minute, and the next two indicate the second. Any further digits indicate fractions of a second. The time, particularly fractions of a second, need not be accurate. It is REQUIRED, however, that any two entries in a dataset changed by successive modifications have strictly ascending modtime values. In addition, each STORE command within a dataset (including simultaneous stores from different connections) MUST use different modtime values. This attribute has enforced validation, so any attempt to STORE a value in this attribute MAY result in a NO response with an INVALID response code. subdataset If this attribute is set, it indicates the existence of a sub- dataset of this entry. Newman & Myers Standards Track [Page 11] RFC 2244 ACAP November 1997 The value consists of a list of relative ACAP URLs (see section 3.2) which may be used to locate the sub-dataset. The base URL is the full path to the entry followed by a slash ("/"). The value "." indicates a subdataset is located directly under this one. Multiple values indicate replicated copies of the subdataset. For example, if the dataset "/folder/site/" has an entry "public-folder" with a subdataset attribute of ".", then there exists a dataset "/folder/site/public-folder/". If the value of the subdataset attribute was instead "//other.acap.domain//folder/site/public-folder/", that would indicate the dataset is actually located on a different ACAP server. A dataset can be created by storing a "subdataset" attribute including ".", and a sub-hierarchy of datasets is deleted by storing a NIL value to the "subdataset" attribute on the entry in the parent dataset. This attribute has enforced syntax validation. Specifically, if an attempt is made to STORE a non-list value (other than NIL), an empty list, or one of the values does not follow the URL syntax rules [BASIC-URL, REL-URL], then this will result in a NO response with an INVALID response code. 3.1.2. Attribute Metadata Each attribute is made up of metadata items which describe that attribute, its value and any associated access controls. Metadata items may be either read-only, in which case the client is never permitted to modify the item, or read-write, in which case the client may modify the item if the access control list (ACL) permits. The following metadata items are defined in this specification: acl The access control list for the attribute, if one exists. If the attribute does not have an ACL, NIL is returned. Read-write. See section 3.5 for the contents of an ACL. attribute The attribute name. Read-only. myrights The set of rights that the client has to the attribute. Read-only. See section 3.5 for the possible rights. Newman & Myers Standards Track [Page 12] RFC 2244 ACAP November 1997 size This is the length of the value. In the case of a multi-value, this is a list of lengths for each of the values. Read-only. value The value. For a multi-value, this is a list of single values. Read-write. Additional items of metadata may be defined in extensions to this protocol. Servers MUST respond to unrecognized metadata by returning a BAD command completion result. 3.2. ACAP URL scheme ACAP URLs are used within the ACAP protocol for the "subdataset" attribute, referrals and inheritance. They provide a convenient syntax for referring to other ACAP datasets. The ACAP URL follows the common Internet scheme syntax as defined in [BASIC-URL] except that plaintext passwords are not permitted. If : is omitted, the port defaults to 674. An ACAP URL has the following general form: url-acap = "acap://" url-server "/" url-enc-entry [url-filter] [url-extension] The element includes the hostname, and optional user name, authentication mechanism and port number. The element contains the name of an entry path encoded according to the rules in [BASIC-URL]. The element is an optional list of interesting attribute names. If omitted, the URL refers to all attributes of the named entry. The element is reserved for extensions to this URL scheme. Note that unsafe or reserved characters such as " " or "?" MUST be hex encoded as described in the URL specification [BASIC-URL]. Hex encoded octets are interpreted according to UTF-8 [UTF8]. 3.2.1. ACAP URL User Name and Authentication Mechanism A user name and/or authentication mechanism may be supplied. They are used in the "AUTHENTICATE" command after making the connection to the ACAP server. If no user name or authentication mechanism is supplied, then the SASL ANONYMOUS [SASL-ANON] mechanism is used by default. If an authentication mechanism is supplied without a user Newman & Myers Standards Track [Page 13] RFC 2244 ACAP November 1997 name, then one SHOULD be obtained from the specified mechanism or requested from the user as appropriate. If a user name is supplied without an authentication mechanism then ";AUTH=*" is assumed. The ";AUTH=" authentication parameter is interpreted as described in the IMAP URL Scheme [IMAP-URL]. Note that if unsafe or reserved characters such as " " or ";" are present in the user name or authentication mechanism, they MUST be encoded as described in the URL specification [BASIC-URL]. 3.2.2. Relative ACAP URLs Because ACAP uses "/" as the hierarchy separator for dataset paths, it works well with the relative URL rules defined in the relative URL specification [REL-URL]. The grammar element is considered part of the user name for purposes of resolving relative ACAP URLs. The base URL for a relative URL stored in an attribute's value is formed by taking the path to the dataset containing that attribute, appending a "/" followed by the entry name of the entry containing that attribute followed by "/". 3.3. Contexts A context is subset of entries in a dataset or datasets, created by a SEARCH command with a MAKECONTEXT modifier. Context names are client-generated strings and must not start with the slash ('/') character. When a client creates a context, it may request automatic notification of changes. A client may also request enumeration of entries within a context. Enumeration simplifies the implementation of a "virtual scrollbar" by the client. A context exists only within the ACAP session in which it was created. When the connection is closed, all contexts associated with that connection are automatically discarded. A server is required to support at least 100 active contexts within a session. If the server supports a larger limit it must advertise it in a CONTEXTLIMIT capability. Newman & Myers Standards Track [Page 14] RFC 2244 ACAP November 1997 3.4. Comparators A comparator is a named function which takes two input values and can be used to perform one or more of four comparison operations: ordering, equality, prefix and substring matching. The ordering operation is used both for the SORT search modifier and the COMPARE and COMPARESTRICT search keys. Ordering comparators can determine the ordinal precedence of any two values. When used for ordering, a comparator's name can be prefixed with "+" or "-" to indicate that the ordering should be normal order or reversed order respectively. If no prefix is included, "+" is assumed. For the purpose of ordering, a comparator may designate certain values as having an undefined ordinal precedence. Such values always collate with equal value after all other values regardless of whether normal or reversed ordering is used. Unless the comparator definition specifies otherwise, multi-values and NIL values have an undefined ordinal precedence. The equality operation is used for the EQUAL search modifier, and simply determines if the two values are considered equal under the comparator function. When comparing a single value to a multi-value, the two are considered equal if any one of the multiple values is equal to the single value. The prefix match operation is used for the PREFIX search modifier, and simply determines if the search value is a prefix of the item being searched. In the case of prefix search on a multi-value, the match is successful if the value is a prefix of any one of the multiple values. The substring match operation is used for the SUBSTRING search modifier, and simply determines if search value is a substring of the item being searched. In the case of substring search on a multi- value, the match is successful if the value is a substring of any one of the multiple values. Rules for naming and registering comparators will be defined in a future specification. Servers MUST respond to unknown or improperly used comparators with a BAD command completion result. Newman & Myers Standards Track [Page 15] RFC 2244 ACAP November 1997 The following comparators are defined by this standard and MUST be implemented: i;octet Operations: Ordering, Equality, Prefix match, Substring match For collation, the i;octet comparator interprets the value of an attribute as a series of unsigned octets with ordinal values from 0 to 255. When ordering two strings, each octet pair is compared in sequence until the octets are unequal or the end of the string is reached. When collating two strings where the shorter is a prefix of the longer, the shorter string is interpreted as having a smaller ordinal value. The "i;octet" or "+i;octet" forms collate smaller ordinal values earlier, and the "-i;octet" form collates larger ordinal values earlier. For the equality function, two strings are equal if they are the same length and contain the same octets in the same order. NIL is equal only to itself. For non-binary, non-nil single values, i;octet ordering is equivalent to the ANSI C [ISO-C] strcmp() function applied to C string representations of the values. For non-binary, non-nil single values, i;octet substring match is equivalent to the ANSI C strstr() function applied to the C string representations of the values. i;ascii-casemap Operations: Ordering, Equality, Prefix match, Substring match The i;ascii-casemap comparator first applies a mapping to the attribute values which translates all US-ASCII letters to uppercase (octet values 0x61 to 0x7A are translated to octet values 0x41 to 0x5A respectively), then applies the i;octet comparator as described above. With this function the values "hello" and "HELLO" have the same ordinal value and are considered equal. i;ascii-numeric Operations: Ordering, Equality The i;ascii-numeric comparator interprets strings as decimal positive integers represented as US-ASCII digits. All values which do not begin with a US-ASCII digit are considered equal with an ordinal value higher than all non-NIL single-valued Newman & Myers Standards Track [Page 16] RFC 2244 ACAP November 1997 attributes. Otherwise, all US-ASCII digits (octet values 0x30 to 0x39) are interpreted starting from the beginning of the string to the first non-digit or the end of the string. 3.5. Access Control Lists (ACLs) An access control list is a set of identifier, rights pairs used to restrict access to a given dataset, attribute or attribute within an entry. An ACL is represented by a multi-value with each value containing an identifier followed by a tab character followed by the rights. The syntax is defined by the "acl" rule in the formal syntax in section 8. Identifier is a UTF-8 string. The identifier "anyone" is reserved to refer to the universal identity (all authentications, including anonymous). All user name strings accepted by the AUTHENTICATE command to authenticate to the ACAP server are reserved as identifiers for the corresponding user. Identifiers starting with a slash ("/") character are reserved for authorization groups which will be defined in a future specification. Identifiers MAY be prefixed with a dash ("-") to indicate a revocation of rights. All other identifiers have implementation-defined meanings. Rights is a string listing a (possibly empty) set of alphanumeric characters, each character listing a set of operations which is being controlled. Letters are reserved for "standard" rights, listed below. The set of standard rights may only be extended by a standards-track or IESG approved experimental RFC. Digits are reserved for implementation or site defined rights. The currently defined standard rights are: x - search (use EQUAL search key with i;octet comparator) r - read (access with SEARCH command) w - write (modify with STORE command) i - insert (perform STORE on a previously NIL value) a - administer (perform SETACL or STORE on ACL attribute/metadata) An implementation may force rights to always or never be granted. In particular, implementations are expected to grant implicit read and administer rights to a user's personal dataset storage in order to avoid denial of service problems. Rights are never tied, unlike the IMAP ACL extension [IMAP-ACL]. It is possible for multiple identifiers in an access control list to apply to a given user (or other authentication identity). For example, an ACL may include rights to be granted to the identifier matching the user, one or more implementation-defined identifiers Newman & Myers Standards Track [Page 17] RFC 2244 ACAP November 1997 matching groups which include the user, and/or the identifier "anyone". These rights are combined by taking the union of all positive rights which apply to a given user and subtracting the union of all negative rights which apply to that user. A client MAY avoid this calculation by using the MYRIGHTS command and metadata items. Each attribute of each entry of a dataset may potentially have an ACL. If an attribute in an entry does not have an ACL, then access is controlled by a default ACL for that attribute in the dataset, if it exists. If there is no default ACL for that attribute in the dataset, access is controlled by a default ACL for that dataset. The default ACL for a dataset must exist. In order to perform any access or manipulation on an entry in a dataset, the client must have 'r' rights on the "entry" attribute of the entry. Implementations should take care not to reveal via error messages the existence of an entry for which the client does not have 'r' rights. A client does not need access to the "subdataset" attribute of the parent dataset in order to access the contents of a dataset. Many of the ACL commands and responses include an "acl object" parameter, for specifying what the ACL applies to. This is a parenthesized list. The list contains just the dataset name when referring to the default ACL for a dataset. The list contains a dataset name and an attribute name when referring to the default ACL for an attribute in a dataset. The list contains a dataset name, an attribute name, and an entry name when referring to the ACL for an attribute of an entry of a dataset. 3.6. Server Response Codes An OK, NO, BAD, ALERT or BYE response from the server MAY contain a response code to describe the event in a more detailed machine parsable fashion. A response code consists of data inside parentheses in the form of an atom, possibly followed by a space and arguments. Response codes are defined when there is a specific action that a client can take based upon the additional information. In order to support future extension, the response code is represented as a slash-separated hierarchy with each level of hierarchy representing increasing detail about the error. Clients MUST tolerate additional hierarchical response code detail which they don't understand. The currently defined response codes are: Newman & Myers Standards Track [Page 18] RFC 2244 ACAP November 1997 AUTH-TOO-WEAK This response code is returned on a tagged NO result from an AUTHENTICATE command. It indicates that site security policy forbids the use of the requested mechanism for the specified authentication identity. ENCRYPT-NEEDED This response code is returned on a tagged NO result from an AUTHENTICATE command. It indicates that site security policy requires the use of a strong encryption mechanism for the specified authentication identity and mechanism. INVALID This response code indicates that a STORE command included data which the server implementation does not permit. It MUST NOT be used unless the dataset class specification for the attribute in question explicitly permits enforced server validation. The argument is the attribute which was invalid. MODIFIED This response code indicates that a conditional store failed because the modtime on the entry is later than the modtime specified with the STORE command UNCHANGEDSINCE modifier. The argument is the entry which had been modified. NOEXIST This response code indicates that a search or NOCREATE store failed because a specified dataset did not exist. The argument is the dataset which does not exist. PERMISSION A command failed due to insufficient permission based on the access control list or implicit rights. The argument is the acl-object which caused the permission failure. QUOTA A STORE or SETACL command which would have increased the size of the dataset failed due to insufficient quota. REFER This response code may be returned in a tagged NO response to any command that takes a dataset name as a parameter. It has one or more arguments with the syntax of relative URLs. It is a referral, indicating that the command should be retried using one of the relative URLs. Newman & Myers Standards Track [Page 19] RFC 2244 ACAP November 1997 SASL This response code can occur in the tagged OK response to a successful AUTHENTICATE command and includes the optional final server response data from the server as specified by SASL [SASL]. TOOMANY This response code may be returned in a tagged OK response to a SEARCH command which includes the LIMIT modifier. The argument returns the total number of matching entries. TOOOLD The modtime specified in the DELETEDSINCE command is too old, so deletedsince information is no longer available. TRANSITION-NEEDED This response code occurs on a NO response to an AUTHENTICATE command. It indicates that the user name is valid, but the entry in the authentication database needs to be updated in order to permit authentication with the specified mechanism. This can happen if a user has an entry in a system authentication database such as Unix /etc/passwd, but does not have credentials suitable for use by the specified mechanism. TRYLATER A command failed due to a temporary server failure. The client MAY continue using local information and try the command later. TRYFREECONTEXT This response code may be returned in a tagged NO response to a SEARCH command which includes the MAKECONTEXT modifier. It indicates that a new context may not be created due to the server's limit on the number of existing contexts. WAYTOOMANY This response code may be returned in a tagged NO response to a SEARCH command which includes a HARDLIMIT search modifier. It indicates that the SEARCH would have returned more entries than the HARDLIMIT permitted. Additional response codes MUST be registered with IANA according to the proceedures in section 7.2. Client implementations MUST tolerate response codes that they do not recognize. Newman & Myers Standards Track [Page 20] RFC 2244 ACAP November 1997 4. Namespace Conventions 4.1. Dataset Namespace The dataset namespace is a slash-separated hierarchy. The first component of the dataset namespace is a dataset class. Dataset classes MUST have a vendor prefix (vendor.) or be specified in a standards track or IESG approved experimental RFC. See section 7.3 for the registration template. The second component of the dataset name is "site", "group", "host", or "user" referring to server-wide data, administrative group data, per-host data and per-user data respectively. For "group", "host", and "user" areas, the third component of the path is the group name, the fully qualified host domain name, or the user name. A path of the form "//~/" is a convenient abbreviation for "//user//". Dataset names which begin with "/byowner/" are reserved as an alternate view of the namespace. This provides a way to see all the dataset classes which a particular owner uses. For example, "/byowner/~//" is an alternate name for "//~/". Byowner provides a way to view a list of dataset classes owned by a given user; this is done using the dataset "/byowner/user//" with the NOINHERIT SEARCH modifier. The dataset "/" may be used to find all dataset classes visible to the current user. A dataset of the form "//user/" may be used to find all users which have made a dataset or entry of that class visible to the current user. The formal syntax for a dataset name is defined by the "dataset-name" rule in section 4.3. 4.2. Attribute Namespace Attribute names which do not contain a dot (".") are reserved for standardized attributes which have meaning in any dataset. In order to simplify client implementations, the attribute namespace is intended to be unique across all datasets. To achieve this, attribute names are prefixed with the dataset class name followed by a dot ("."). Attributes which affect management of the dataset are prefixed with "dataset.". In addition, a subtree of the "vendor." attribute namespace may be registered with IANA according to the rules in section 7.4. ACAP implementors are encouraged to help define interoperable dataset classes specifications rather than using the private attribute namespace. Newman & Myers Standards Track [Page 21] RFC 2244 ACAP November 1997 Some users or sites may wish to add their own private attributes to certain dataset classes. In order to enable this, the "user.." and "site." subtrees of the attribute namespace are reserved for user-specific and site-specific attributes respectively and will not be standardized. Such attributes are not interoperable so are discouraged in favor of defining standard attributes. A future extension is expected to permit discovery of syntax for user or site-specific attributes. Clients wishing to support display of user or site-specific attributes should display the value of any non-NIL single-valued "user.." or "site." attribute which has valid UTF-8 syntax. The formal syntax for an attribute name is defined by the "attribute-name" rule in the next section. 4.3. Formal Syntax for Dataset and Attribute Namespace The naming conventions for datasets and attributes are defined by the following ABNF. Note that this grammar is not part of the ACAP protocol syntax in section 8, as dataset names and attribute names are encoded as strings within the ACAP protocol. attribute-dacl = "dataset.acl" *("." name-component) attribute-dset = dataset-std 1*("." name-component) ;; MUST be defined in a dataset class specification attribute-name = attribute-std / attr-site / attr-user / vendor-name attribute-std = "entry" / "subdataset" / "modtime" / "dataset.inherit" / attribute-dacl / attribute-dset attr-site = "site" 1*("." name-component) attr-user = "user." name-component 1*("." name-component) byowner = "/byowner/" owner "/" [dataset-class "/" dataset-sub] dataset-class = dataset-std / vendor-name dataset-normal = "/" [dataset-class "/" (owner-prefix / dataset-tail)] dataset-name = byowner / dataset-normal Newman & Myers Standards Track [Page 22] RFC 2244 ACAP November 1997 dataset-std = name-component ;; MUST be registered with IANA and the spec MUST ;; be published as a standards track or ;; IESG-approved experimental RFC dataset-sub = *(dname-component "/") ;; The rules for this portion of the namespace may ;; be further restricted by the dataset class ;; specification. dataset-tail = owner "/" dataset-sub dname-component = 1*UTF8-CHAR ;; MUST NOT begin with "." or contain "/" name-component = 1*UTF8-CHAR ;; MUST NOT contain ".", "/", "%", or "*" owner = "site" / owner-host / owner-group / owner-user / "~" owner-group = "group/" dname-component owner-host = "host/" dname-component owner-prefix = "group/" / "host/" / "user/" owner-user = "user/" dname-component vendor-name = vendor-token *("." name-component) vendor-token = "vendor." name-component ;; MUST be registered with IANA 5. Dataset Management The entry with an empty name ("") in the dataset is used to hold management information for the dataset as a whole. 5.1. Dataset Inheritance It is possible for one dataset to inherit data from another. The dataset from which the data is inherited is called the base dataset. Data in the base dataset appears in the inheriting dataset, except when overridden by a STORE to the inheriting dataset. Newman & Myers Standards Track [Page 23] RFC 2244 ACAP November 1997 The base dataset is usually a system-wide or group-wide set of defaults. A system-wide dataset usually has one inheriting dataset per user, allowing each user to add to or modify the defaults as appropriate. An entry which exists in both the inheriting and base dataset inherits a modtime equal to the greater of the two modtimes. An attribute in such an entry is inherited from the base dataset if it was never modified by a STORE command in the inheriting dataset or if DEFAULT was stored to that attribute. This permits default entries to be amended rather than replaced in the inheriting dataset. The "subdataset" attribute is not directly inherited. If the base dataset includes a "subdataset" attribute and the inheriting dataset does not, then the "subdataset" attribute will inherit a virtual value of a list containing a ".". The subdataset at that node is said to be a "virtual" dataset as it is simply a virtual copy of the appropriate base dataset with all "subdataset" attributes changed to a list containing a ".". A virtual dataset is not visible if NOINHERIT is specified on the SEARCH command. Servers MUST support at least two levels of inheritance. This permits a user's dataset such as "/options/user/fred/common" to inherit from a group dataset such as "/options/group/dinosaur operators/common" which in turn inherits from a server-wide dataset such as "/options/site/common". 5.2. Dataset Attributes The following attributes apply to management of the dataset when stored in the "" entry of a dataset. These attributes are not inherited. dataset.acl This holds the default access control list for the dataset. This attribute is validated, so an invalid access control list in a STORE command will result in a NO response with an INVALID response code. dataset.acl. This holds the default access control list for an attribute within the dataset. This attribute is validated, so an invalid access control list in a STORE command will result in a NO response with an INVALID response code. dataset.inherit This holds the name of a dataset from which to inherit according to the rules in the previous section. This attribute MAY refer Newman & Myers Standards Track [Page 24] RFC 2244 ACAP November 1997 to a non-existent dataset, in which case nothing is inherited. This attribute is validated, so illegal dataset syntax or an attempt to store a multi-value will result in a NO response with an INVALID response code. 5.3. Dataset Creation When a dataset is first created (by storing a "." in the subdataset attribute or storing an entry in a previously non-existent dataset), the dataset attributes are initialized with the values from the parent dataset in the "/byowner/" hierarchy. In the case of the "dataset.inherit" attribute, the appropriate hierarchy component is added. For example, given the following entry (note that \t refers to the US-ASCII horizontal tab character): entry path "/byowner/user/joe/" dataset.acl ("joe\txrwia" "fred\txr") dataset.inherit "/byowner/site" If a new dataset class "/byowner/user/joe/new" is created, it will have the following dataset attributes: entry path "/byowner/user/joe/new/" dataset.acl ("joe\txrwia" "fred\txr") dataset.inherit "/byowner/site/new" Note that the dataset "/byowner/user/joe/new/" is equivalent to "/new/user/joe/". 5.4. Dataset Class Capabilities Certain dataset classes or dataset class features may only be useful if there is an active updating client or integrated server support for the feature. The dataset class "capability" is reserved to allow clients or servers to advertise such features. The "entry" attribute within this dataset class is the name of the dataset class whose features are being described. The attributes are prefixed with "capability.." and are defined by the appropriate dataset class specification. Since it is possible for an unprivileged user to run an active client for himself, a per-user capability dataset is useful. The dataset "/capability/~/" holds information about all features available to the user (via inheritance), and the dataset "/capability/site/" holds information about all features supported by the site. Newman & Myers Standards Track [Page 25] RFC 2244 ACAP November 1997 5.5. Dataset Quotas Management and scope of quotas is implementation dependent. Clients can check the applicable quota limit and usage (in bytes) with the GETQUOTA command. Servers can notify the client of a low quota situation with the QUOTA untagged response. 6. Command and Response Specifications ACAP commands and responses are described in this section. Commands are organized first by the state in which the command is permitted, then by a general category of command type. Command arguments, identified by "Arguments:" in the command descriptions below, are described by function, not by syntax. The precise syntax of command arguments is described in the Formal Syntax section. Some commands cause specific server data to be returned; these are identified by "Data:" in the command descriptions below. See the response descriptions in the Responses section for information on these responses, and the Formal Syntax section for the precise syntax of these responses. It is possible for server data to be transmitted as a result of any command; thus, commands that do not specifically require server data specify "no specific data for this command" instead of "none". The "Result:" in the command description refers to the possible tagged status responses to a command, and any special interpretation of these status responses. 6.1. Initial Connection Upon session startup, the server sends one of two untagged responses: ACAP or BYE. The untagged BYE response is described in section 6.2.8. 6.1.1. ACAP Untagged Response Data: capability list The untagged ACAP response indicates the session is ready to accept commands and contains a space-separated listing of capabilities that the server supports. Each capability is represented by a list containing the capability name optionally followed by capability specific string arguments. Newman & Myers Standards Track [Page 26] RFC 2244 ACAP November 1997 ACAP capability names MUST be registered with IANA according to the rules in section 7.1. Client implementations SHOULD NOT require any capability name beyond those defined in this specification, and MUST tolerate any unknown capability names. A client implementation MAY be configurable to require SASL mechanisms other than CRAM-MD5 [CRAM-MD5] for site security policy reasons. The following initial capabilities are defined: CONTEXTLIMIT The CONTEXTLIMIT capability has one argument which is a number describing the maximum number of contexts the server supports per connection. The number 0 indicates the server has no limit, otherwise this number MUST be greater than 100. IMPLEMENTATION The IMPLEMENTATION capability has one argument which is a string describing the server implementation. ACAP clients MUST NOT alter their behavior based on this value. It is intended primarily for debugging purposes. SASL The SASL capability includes a list of the authentication mechanisms supported by the server. See section 6.3.1. Example: S: * ACAP (IMPLEMENTATION "ACME v3.5") (SASL "CRAM-MD5") (CONTEXTLIMIT "200") 6.2. Any State The following commands and responses are valid in any state. 6.2.1. NOOP Command Arguments: none Data: no specific data for this command (but see below) Result: OK - noop completed BAD - command unknown or arguments invalid The NOOP command always succeeds. It does nothing. It can be used to reset any inactivity auto-logout timer on the server. Example: C: a002 NOOP Newman & Myers Standards Track [Page 27] RFC 2244 ACAP November 1997 S: a002 OK "NOOP completed" 6.2.2. LANG Command Arguments: list of language preferences Data: intermediate response: LANG Result: OK - lang completed NO - no matching language available BAD - command unknown or arguments invalid One or more arguments are supplied to indicate the client's preferred languages [LANG-TAGS] for error messages. The server will match each client preference in order against its internal table of available error string languages. For a client preference to match a server language, the client's language tag MUST be a prefix of the server's tag and match up to a "-" or the end of string. If a match is found, the server returns an intermediate LANG response and an OK response. The LANG response indicates the actual language selected and appropriate comparators for use with the languages listed in the LANG command. If no LANG command is issued, all error text strings MUST be in the registered language "i-default" [CHARSET-LANG-POLICY], intended for an international audience. Example: C: A003 LANG "fr-ca" "fr" "en-ca" "en-uk" S: A003 LANG "fr-ca" "i;octet" "i;ascii-numeric" "i;ascii-casemap" "en;primary" "fr;primary" S: A003 OK "Bonjour" 6.2.3. LANG Intermediate Response Data: language for error responses appropriate comparators The LANG response indicates the language which will be used for error responses and the comparators which are appropriate for the languages listed in the LANG command. The comparators SHOULD be in approximate order from most efficient (usually "i;octet") to most appropriate for human text in the preferred language. Newman & Myers Standards Track [Page 28] RFC 2244 ACAP November 1997 6.2.4. LOGOUT Command Arguments: none Data: mandatory untagged response: BYE Result: OK - logout completed BAD - command unknown or arguments invalid The LOGOUT command informs the server that the client is done with the session. The server must send a BYE untagged response before the (tagged) OK response, and then close the network connection. Example: C: A023 LOGOUT S: * BYE "ACAP Server logging out" S: A023 OK "LOGOUT completed" (Server and client then close the connection) 6.2.5. OK Response Data: optional response code human-readable text The OK response indicates an information message from the server. When tagged, it indicates successful completion of the associated command. The human-readable text may be presented to the user as an information message. The untagged form indicates an information-only message; the nature of the information MAY be indicated by a response code. Example: S: * OK "Master ACAP server is back up" 6.2.6. NO Response Data: optional response code human-readable text The NO response indicates an operational error message from the server. When tagged, it indicates unsuccessful completion of the associated command. The untagged form indicates a warning; the command may still complete successfully. The human-readable text describes the condition. Example: C: A010 SEARCH "/addressbook/" DEPTH 3 RETURN ("*") EQUAL "entry" "+i;octet" "bozo" S: * NO "Master ACAP server is down, your data may Newman & Myers Standards Track [Page 29] RFC 2244 ACAP November 1997 be out of date." S: A010 OK "search done" ... C: A222 STORE ("/folder/site/comp.mail.misc" "folder.creation-time" "19951206103412") S: A222 NO (PERMISSION ("/folder/site/")) "Permission denied" 6.2.7. BAD Response Data: optional response code human-readable text The BAD response indicates an error message from the server. When tagged, it reports a protocol-level error in the client's command; the tag indicates the command that caused the error. The untagged form indicates a protocol-level error for which the associated command can not be determined; it may also indicate an internal server failure. The human-readable text describes the condition. Example: C: ...empty line... S: * BAD "Empty command line" C: A443 BLURDYBLOOP S: A443 BAD "Unknown command" C: A444 NOOP Hello S: A444 BAD "invalid arguments" 6.2.8. BYE Untagged Response Data: optional response code human-readable text The untagged BYE response indicates that the server is about to close the connection. The human-readable text may be displayed to the user in a status report by the client. The BYE response may be sent as part of a normal logout sequence, or as a panic shutdown announcement by the server. It is also used by some server implementations as an announcement of an inactivity auto- logout. This response is also used as one of two possible greetings at session startup. It indicates that the server is not willing to accept a session from this client. Example: S: * BYE "Auto-logout; idle for too long" Newman & Myers Standards Track [Page 30] RFC 2244 ACAP November 1997 6.2.9. ALERT Untagged Response Data: optional response code human-readable text The human-readable text contains a special human generated alert message that MUST be presented to the user in a fashion that calls the user's attention to the message. This is intended to be used for vital messages from the server administrator to the user, such as a warning that the server will soon be shut down for maintenance. Example: S: * ALERT "This ACAP server will be shut down in 10 minutes for system maintenance." 6.3. Non-Authenticated State In non-authenticated state, the AUTHENTICATE command establishes authentication and enters authenticated state. The AUTHENTICATE command provides a general mechanism for a variety of authentication techniques. Server implementations may allow non-authenticated access to certain information by supporting the SASL ANONYMOUS [SASL-ANON] mechanism. Once authenticated (including as anonymous), it is not possible to re-enter non-authenticated state. Only the any-state commands (NOOP, LANG and LOGOUT) and the AUTHENTICATE command are valid in non-authenticated state. 6.3.1. AUTHENTICATE Command Arguments: SASL mechanism name optional initial response Data: continuation data may be requested Result: OK - authenticate completed, now in authenticated state NO - authenticate failure: unsupported authentication mechanism, credentials rejected BAD - command unknown or arguments invalid, authentication exchange cancelled Newman & Myers Standards Track [Page 31] RFC 2244 ACAP November 1997 The AUTHENTICATE command indicates a SASL [SASL] authentication mechanism to the server. If the server supports the requested authentication mechanism, it performs an authentication protocol exchange to authenticate and identify the user. Optionally, it also negotiates a security layer for subsequent protocol interactions. If the requested authentication mechanism is not supported, the server rejects the AUTHENTICATE command by sending a tagged NO response. The authentication protocol exchange consists of a series of server challenges and client answers that are specific to the authentication mechanism. A server challenge consists of a command continuation request with the "+" token followed by a string. The client answer consists of a line consisting of a string. If the client wishes to cancel an authentication exchange, it should issue a line with a single unquoted "*". If the server receives such an answer, it must reject the AUTHENTICATE command by sending a tagged BAD response. The optional initial-response argument to the AUTHENTICATE command is used to save a round trip when using authentication mechanisms that are defined to send no data in the initial challenge. When the initial-response argument is used with such a mechanism, the initial empty challenge is not sent to the client and the server uses the data in the initial-response argument as if it were sent in response to the empty challenge. If the initial-response argument to the AUTHENTICATE command is used with a mechanism that sends data in the initial challenge, the server rejects the AUTHENTICATE command by sending a tagged NO response. The service name specified by this protocol's profile of SASL is "acap". If a security layer is negotiated through the SASL authentication exchange, it takes effect immediately following the CRLF that concludes the authentication exchange for the client, and the CRLF of the tagged OK response for the server. All ACAP implementations MUST implement the CRAM-MD5 SASL mechanism [CRAM-MD5], although they MAY offer a configuration option to disable it if site security policy dictates. The example below is the same example described in the CRAM-MD5 specification. If an AUTHENTICATE command fails with a NO response, the client may try another authentication mechanism by issuing another AUTHENTICATE command. In other words, the client may request authentication types in decreasing order of preference. Newman & Myers Standards Track [Page 32] RFC 2244 ACAP November 1997 Example: S: * ACAP (IMPLEMENTATION "Blorfysoft v3.5") (SASL "CRAM-MD5" "KERBEROS_V4") C: A001 AUTHENTICATE "CRAM-MD5" S: + "<1896.697170952@postoffice.reston.mci.net>" C: "tim b913a602c7eda7a495b4e6e7334d3890" S: A001 OK "CRAM-MD5 authentication successful" 6.4. Searching This section describes the SEARCH command, for retrieving data from datasets. 6.4.1. SEARCH Command Arguments: dataset or context name optional list of modifiers search criteria Data: intermediate responses: ENTRY, MODTIME, REFER untagged responses: ADDTO, REMOVEFROM, CHANGE, MODTIME Result: OK - search completed NO - search failure: can't perform search BAD - command unknown or arguments invalid The SEARCH command identifies a subset of entries in a dataset and returns information on that subset to the client. Inherited entries and attributes are included in the search unless the NOINHERIT search modifier is included or the user does not have permission to read the attributes in the base dataset. The first argument to SEARCH identifies what is to be searched. If the string begins with a slash ("/"), it is the name of a dataset to be searched, otherwise it is a name of a context that was created by a SEARCH command given previously in the session. A successful SEARCH command MAY result in intermediate ENTRY responses and MUST result in a MODTIME intermediate response. Following that are zero or more modifiers to the search. Each modifier may be specified at most once. The defined modifiers are: Newman & Myers Standards Track [Page 33] RFC 2244 ACAP November 1997 DEPTH number The SEARCH command will traverse the dataset tree up to the specified depth. ENTRY responses will include the full path to the entry. A value of "0" indicates that the search should traverse the entire tree. A value of "1" is the default and indicates only the specified dataset should be searched. If a dataset is traversed which is not located on the current server, then a REFER intermediate response is returned for that subtree and the search continues. HARDLIMIT number If the SEARCH command would result in more than number entries, the SEARCH fails with a NO completion result with a WAYTOOMANY response code. LIMIT number number Limits the number of intermediate ENTRY responses that the search may generate. The first numeric argument specifies the limit, the second number specifies the number of entries to return if the number of matches exceeds the limit. If the limit is exceeded, the SEARCH command still succeeds, returning the total number of matches in a TOOMANY response code in the tagged OK response. MAKECONTEXT [ENUMERATE] [NOTIFY] context Causes the SEARCH command to create a context with the name given in the argument to refer to the matching entries. If the SEARCH is successful, the context name may then be given as an argument to subsequent SEARCH commands to search the set of matching entries. If a context with the specified name already exists, it is first freed. If a new context may not be created due to the server's limit on the number of existing contexts, the command fails, returning a TRYFREECONTEXT response code in the NO completion response. The optional "ENUMERATE" and "NOTIFY" arguments may be included to request enumeration of the context (for virtual scroll bars) or change notifications for the context. If "NOTIFY" is not requested, the context represents a snapshot of the entries at the time the SEARCH was issued. ENUMERATE requests that the contents of the context be ordered according to the SORT modifier and that sequential numbers, starting with one, be assigned to the entries in the context. This permits the RANGE modifier to be used to fetch portions of the ordered context. Newman & Myers Standards Track [Page 34] RFC 2244 ACAP November 1997 NOTIFY requests that the server send untagged ADDTO, REMOVEFROM, CHANGE, and MODTIME responses while the context created by this SEARCH command exists. The server MAY issue untagged ADDTO, REMOVEFROM, CHANGE and MODTIME notifications for a context at any time between the issuing of the SEARCH command with MAKECONTEXT NOTIFY and the completion of a FREECONTEXT command for the context. Notifications are only issued for changes which occur after the server receives the SEARCH command which created the context. After issuing a sequence of ADDTO, REMOVEFROM or CHANGE notifications, the server MUST issue an untagged MODTIME notification indicating that the client has all updates to the entries in the context up to and including the given modtime value. Servers are permitted a reasonable delay to batch change notifications before sending them to the client. The position arguments of the ADDTO, REMOVEFROM and CHANGE notifications are 0 if ENUMERATE is not requested. NOINHERIT This causes the SEARCH command to operate without inheritance. It can be used to tell which values are explicit overrides. If MAKECONTEXT is also specified, the created context is also not affected by inheritance. RETURN (metadata...) Specifies what is to be returned in intermediate ENTRY responses. If this modifier is not specified, no intermediate ENTRY responses are returned. Inside the parentheses is an optional list of attributes, each optionally followed by a parenthesized list of metadata. If the parenthesized list of metadata is not specified, it defaults to "(value)". An attribute name with a trailing "*" requests all attributes with that prefix. A "*" by itself requests all attributes. If the parenthesized list of metadata is not specified for an attribute with a trailing "*", it defaults to "(attribute value)". Results matching such an attribute pattern are grouped in parentheses. Following the last intermediate ENTRY response, the server returns a single intermediate MODTIME response. Newman & Myers Standards Track [Page 35] RFC 2244 ACAP November 1997 SORT (attribute comparator ...) Specifies the order in which any resulting ENTRY replies are to be returned to the client. The SORT modifier takes as an argument a parenthesized list of one or more attribute/comparator pairs. Attribute lists the attribute to sort on, comparator specifies the name of the collation rule to apply to the values of the attribute. Successive attribute/comparator pairs are used to order two entries only when all preceding pairs indicate the two entries collate the same. If the SORT modifier is used in conjunction with the MAKECONTEXT modifier, the SORT modifier specifies the ordering of entries in the created context. If no SORT modifier is specified, or none of the attribute/comparator pairs indicates an order for the two entries, the server uses the order of the entries that exists in the context or dataset being searched. Following the modifiers is the search criteria. Searching criteria consist of one or more search keys. Search keys may be combined using the AND, and OR search keys. For example, the criteria (the newline is for readability and not part of the criteria): AND COMPARE "modtime" "+i;octet" "19951206103400" COMPARE "modtime" "-i;octet" "19960112000000" refers to all entries modified between 10:34 December 6 1995 and midnight January 12, 1996 UTC. The currently defined search keys are as follows. ALL This matches all entries. AND search-key1 search-key2 Entries that match both search keys. COMPARE attribute comparator value Entries for which the value of the specified attribute collates using the specified comparator the same or later than the specified value. COMPARESTRICT attribute comparator value Entries for which the specified attribute collates using the specified comparator later than the specified value. Newman & Myers Standards Track [Page 36] RFC 2244 ACAP November 1997 EQUAL attribute comparator value Entries for which the value of the attribute is equal to the specified value using the specified comparator. NOT search-key Entries that do not match the specified search key. OR search-key1 search-key2 Entries that match either search key. PREFIX attribute comparator value Entries which begin with the specified value using the specified comparator. If the specified comparator doesn't support substring matching, a BAD response is returned. RANGE start end time Entries which are within the specified range of the enumerated context's ordering. The lowest-ordered entry in the context is assigned number one, the next lowest entry is assigned number two, and so on. The numeric arguments specify the lowest and highest numbers to match. The time specifies that the client has processed notifications for the context up to the specified time. If the context has been modified since then, the server MUST either return a NO with a MODIFIED response code, or return the results that the SEARCH would have returned if none of the changes since that time had been made. RANGE is only permitted on enumerated contexts. If RANGE is used with a dataset or non-enumerated context, the server MUST return a BAD response. SUBSTRING attribute comparator value Entries which contain the specified value, using the specified comparator. If the specified comparator doesn't support substring matching, a BAD response is returned. 6.4.2. ENTRY Intermediate Response Data: entry name entry data The ENTRY intermediate response occurs as a result of a SEARCH or STORE command. This is the means by which dataset entries are returned to the client. Newman & Myers Standards Track [Page 37] RFC 2244 ACAP November 1997 The ENTRY response begins with the entry name, if a SEARCH command without the DEPTH modifier was issued, or the entry path in other cases. This is followed by a set of zero or more items, one for each metadata item in the RETURN search modifier. Results matching an attribute pattern or returning multiple metadata items are grouped in parentheses. 6.4.3. MODTIME Intermediate Response Data: modtime value The MODTIME intermediate response occurs as a result of a SEARCH command. It indicates that the just created context or the previously returned ENTRY responses include all updates to the returned entries up to and including the modtime value in the argument. 6.4.4. REFER Intermediate Response Data: dataset path relative ACAP URLs The REFER intermediate response occurs as a result of a multi-level SEARCH where one of the levels is located on a different server. The response indicates the dataset which is not located on the current server and one or more relative ACAP URLs for where that dataset may be found. 6.4.5. Search Examples Here are some SEARCH command exchanges between the client and server: C: A046 SEARCH "/addressbook/" DEPTH 3 RETURN ("addressbook.Alias" "addressbook.Email" "addressbook.List") OR NOT EQUAL "addressbook.Email" "i;octet" NIL NOT EQUAL "addressbook.List" "i;octet" NIL S: A046 ENTRY "/addressbook/user/joe/A0345" "fred" "fred@stone.org" NIL S: A046 ENTRY "/addressbook/user/fred/A0537" "joe" "joe@stone.org" NIL S: A046 ENTRY "/addressbook/group/Dinosaur Operators/A423" "saurians" NIL "1" S: A046 MODTIME "19970728105252" S: A046 OK "SEARCH completed" C: A047 SEARCH "/addressbook/user/fred/" RETURN ("*") EQUAL "entry" "i;octet" "A0345" S: A047 ENTRY "A0345" (("modtime" "19970728102226") Newman & Myers Standards Track [Page 38] RFC 2244 ACAP November 1997 ("addressbook.Alias" "fred") ("addressbook.Email" "fred@stone.org") ("addressbook.CommonName" "Fred Flintstone") ("addressbook.Surname" "Flintstone") ("addressbook.GivenName" "Fred")) S: A047 MODTIME "19970728105258" S: A047 OK "SEARCH completed" C: A048 SEARCH "/options/~/vendor.example/" RETURN ("option.value"("size" "value" "myrights")) SORT ("entry" "i;octet") COMPARE "modtime" "i;octet" "19970727123225" S: A048 ENTRY "blurdybloop" (5 "ghoti" "rwia") S: A048 ENTRY "buckybits" (2 "10" "rwia") S: A048 ENTRY "windowSize" (7 "100x100" "rwia") S: A048 MODTIME "19970728105304" S: A048 OK "SEARCH completed" C: A049 SEARCH "/addressbook/~/public" RETURN ("addressbook.Alias" "addressbook.Email") MAKECONTEXT ENUMERATE "blob" LIMIT 100 1 SORT ("addressbook.Alias" "i;octet") NOT EQUAL "addressbook.Email" NIL S: A049 ENTRY "A437" "aaguy" "aaguy@stone.org" S: A049 MODTIME "19970728105308" S: A049 OK (TOOMANY 347) "Context 'blob' created" C: A050 SEARCH "blob" RANGE 2 2 "19970728105308" ALL S: A050 ENTRY "A238" "abguy" "abguy@stone.org" S: A050 MODTIME "19970728105310" S: A050 OK "SEARCH Completed" 6.5. Contexts The following commands use contexts created by a SEARCH command with a MAKECONTEXT modifier. 6.5.1. FREECONTEXT Command Arguments: context name Data: no specific data for this command Result: OK - freecontext completed NO - freecontext failure: no such context BAD - command unknown or arguments invalid Newman & Myers Standards Track [Page 39] RFC 2244 ACAP November 1997 The FREECONTEXT command causes the server to free all state associated with the named context. The context may no longer be searched and the server will no longer issue any untagged responses for the context. The context is no longer counted against the server's limit on the number of contexts. Example: C: A683 FREECONTEXT "blurdybloop" S: A683 OK "Freecontext completed" 6.5.2. UPDATECONTEXT Command Arguments: list of context names Data: untagged responses: ADDTO REMOVEFROM CHANGE MODTIME Result: OK - Updatecontext completed: all updates completed NO - Updatecontext failed: no such context not a notify context BAD - command unknown or arguments invalid The UPDATECONTEXT command causes the server to ensure that the client is notified of all changes known to the server for the contexts listed as arguments up to the current time. The contexts listed in the arguments must have been previously given to a successful SEARCH command with a MAKECONTEXT NOTIFY modifier. A MODTIME untagged response MUST be returned if any read-write metadata in the context changed since the last MODTIME for that context. This includes metadata which is not listed in the RETURN modifier for the context. While a server may issue untagged ADDTO, REMOVEFROM, CHANGE, and MODTIME at any time, the UPDATECONTEXT command is used to "prod" the server to send any notifications it has not sent yet. The UPDATECONTEXT command SHOULD NOT be used to poll for updates. Example: C: Z4S9 UPDATECONTEXT "blurdybloop" "blarfl" S: Z4S9 OK "client has been notified of all changes" 6.5.3. ADDTO Untagged Response Data: context name entry name position metadata list Newman & Myers Standards Track [Page 40] RFC 2244 ACAP November 1997 The untagged ADDTO response informs the client that an entry has been added to a context. The response includes the position number of the added entry (the first entry in the context is numbered 1) and those metadata contained in the entry which match the RETURN statement when the context was created. For enumerated contexts, the ADDTO response implicitly adds one to the position of all members of the context which had position numbers that were greater than or equal to the ADDTO position number. For non-enumerated contexts, the position field is always 0. Example: S: * ADDTO "blurdybloop" "fred" 15 ("addressbook.Email" "fred@stone.org") 6.5.4. REMOVEFROM Untagged Response Data: context name entry name old position The untagged REMOVEFROM response informs the client that an entry has been removed from a context. The response includes the position number that the removed entry used to have (the first entry in the context is numbered 1). For enumerated contexts, the REMOVEFROM response implicitly subtracts one from the position numbers of all members of the context which had position numbers greater than the REMOVEFROM position number. For non-enumerated contexts, the position field is always 0. Example: S: * REMOVEFROM "blurdybloop" "fred" 15 6.5.5. CHANGE Untagged Response Data: context name entry name old position new position metadata list The untagged CHANGE response informs the client that an entry in a context has either changed position in the context or has changed the values of one or more of the attributes specified in the RETURN modifier when the context was created. Newman & Myers Standards Track [Page 41] RFC 2244 ACAP November 1997 The response includes the previous and current position numbers of the entry (which are 0 if ENUMERATE was not specified on the context) and the attribute metadata requested in the RETURN modifier when the context was created. For enumerated contexts, the CHANGE response implicitly changes the position numbers of all entries which had position numbers between the old and new position. If old position is less than new position, than one is subtracted from all entries which had position numbers in that range. Otherwise one is added to all entries which had position numbers in that range. If the old position and new position are the same, then no implicit position renumbering occurs. CHANGE responses are not issued for entries which have changed position implicitly due to another ADDTO, REMOVEFROM or CHANGE response. Example: S: * CHANGE "blurdybloop" "fred" 15 10 ("addressbook.Email" "fred@stone.org") 6.5.6. MODTIME Untagged Response Data: context name modtime value The untagged MODTIME response informs the client that it has received all updates to entries in the context which have modtime values less than or equal to the modtime value in the argument. Example: S: * MODTIME mycontext "19970320162338" 6.6. Dataset modification The following commands and responses handle modification of datasets. Newman & Myers Standards Track [Page 42] RFC 2244 ACAP November 1997 6.6.1. STORE Command Arguments: entry store list Data: intermediate responses: ENTRY Result: OK - store completed NO - store failure: can't store that name UNCHANGEDSINCE specified and entry changed BAD - command unknown or arguments invalid invalid UTF-8 syntax in attribute name Creates, modifies, or deletes the named entries in the named datasets. The values of metadata not specified in the command are not changed. Setting the "value" metadata of an attribute to NIL removes that attribute from the entry. Setting the "value" of the "entry" attribute to NIL removes that entry from the dataset and cancels inheritance for the entire entry. Setting the "value" of the "entry" attribute to DEFAULT removes that entry from the inheriting dataset and reverts the entry and its attributes to inherited values, if any. Changing the value of the "entry" attribute renames the entry. Storing DEFAULT to the "value" metadata of an attribute is equivalent to storing NIL, except that inheritance is enabled for that attribute. If a non-NIL value is inherited then an ENTRY intermediate response is generated to notify the client of the this change. The ENTRY response includes the entry-path and the attribute name and value metadata for each attribute which reverted to a non-NIL inherited setting. Storing NIL to the "value" metadata of an attribute MAY be treated equivalent to storing DEFAULT to that "value" if there is a NIL value in the base dataset. The STORE command is followed by one or more entry store lists. Each entry store list begins with an entry path followed by STORE modifiers, followed by zero or more attribute store items. Each attribute store item is made up of the attribute name followed by NIL (to remove the attribute's value), DEFAULT (to revert the item to any inherited value), a single value (to set the attribute's single value), or a list of metadata items to modify. The following STORE modifiers may be specified: Newman & Myers Standards Track [Page 43] RFC 2244 ACAP November 1997 NOCREATE By default, the server MUST create any datasets necessary to store the entry, including multiple hierarchy levels. If NOCREATE is specified, the STORE command will fail with a NOEXIST error unless the parent dataset already exists. UNCHANGEDSINCE If the "modtime" of the entry is later than the unchangedsince time, then the store fails with a MODIFIED response code. Use of UNCHANGEDSINCE with a time of "00000101000000" will always fail if the entry exists. Clients writing to a shared dataset are encouraged to use UNCHANGEDSINCE when modifying an existing entry. The server MUST either make all the changes specified in a single STORE command or make none of them. If successful, the server MUST update the "modtime" attribute for every entry which was changed. It is illegal to list any metadata item within an attribute twice, any attribute within an entry twice or any entry path twice. The server MUST return a BAD response if this happens. The server MAY re-order the strings in a multi-value on STORE and MAY remove duplicate strings. However, SEARCH MUST return multi- values and the associated size list metadata in a consistant order. Example: C: A342 STORE ("/addressbook/user/fred/ABC547" "addressbook.TelephoneNumber" "555-1234" "addressbook.CommonName" "Barney Rubble" "addressbook.AlternateNames" ("value" ("Barnacus Rubble" "Coco Puffs Thief")) "addressbook.Email" NIL) S: A342 OK "Store completed" C: A343 STORE ("/addressbook/user/joe/ABD42" UNCHANGEDSINCE "19970320162338" "user.joe.hair-length" "10 inches") S: A343 NO (MODIFIED) "'ABD42' has been changed by somebody else." C: A344 STORE ("/addressbook/group/Developers/ACD54" "entry" NIL) S: A344 OK "Store completed" C: A345 STORE ("/option/~/common/SMTPserver" "option.value" DEFAULT) S: A345 ENTRY "/option/~/common/SMTPserver" Newman & Myers Standards Track [Page 44] RFC 2244 ACAP November 1997 "option.value" "smtp.server.do.main" S: A345 OK "Store completed" C: A347 STORE ("/addressbook/~/" "dataset.inherit" "/addressbook/group/Developers") S: A347 OK "Store completed" 6.6.2. DELETEDSINCE Command Arguments: dataset name time Data: intermediate response: DELETED Result: OK - DELETEDSINCE completed NO - DELETEDSINCE failure: can't read dataset date too far in the past BAD - command unknown or arguments invalid The DELETEDSINCE command returns in intermediate DELETED replies the names of entries that have been deleted from the named dataset since the given time. Servers may impose a limit on the number or age of deleted entry names they keep track of. If the server does not have information going back to the specified time, the command fails, returning a TOOOLD response code in the tagged NO response. Example: C: Z4S9 DELETEDSINCE "/folder/site/" 19951205103412 S: Z4S9 DELETED "blurdybloop" S: Z4S9 DELETED "anteaters" S: Z4S9 OK "DELETEDSINCE completed" C: Z4U3 DELETEDSINCE "/folder/site/" 19951009040854 S: Z4U3 NO (TOOOLD) "Don't have that information" 6.6.3. DELETED Intermediate Response Data: entry name The intermediate DELETED response occurs as a result of a DELETEDSINCE command. It returns an entry that has been deleted from the dataset specified in the DELETEDSINCE command. 6.7. Access Control List Commands The commands in this section are used to manage access control lists. Newman & Myers Standards Track [Page 45] RFC 2244 ACAP November 1997 6.7.1. SETACL Command Arguments: acl object authentication identifier access rights Data: no specific data for this command Result: OK - setacl completed NO - setacl failure: can't set acl BAD - command unknown or arguments invalid The SETACL command changes the access control list on the specified object so that the specified identifier is granted the permissions enumerated in rights. If the object did not previously have an access control list, one is created. Example: C: A123 SETACL ("/addressbook/~/public/") "anyone" "r" S: A123 OK "Setacl complete" C: A124 SETACL ("/folder/site/") "B1FF" "rwa" S: A124 NO (PERMISSION ("/folder/site/")) "'B1FF' not permitted to modify access rights for '/folder/site/'" 6.7.2. DELETEACL Command Arguments: acl object optional authentication identifier Data: no specific data for this command Result: OK - deleteacl completed NO - deleteacl failure: can't delete acl BAD - command unknown or arguments invalid If given the optional identifier argument, the DELETEACL command removes any portion of the access control list on the specified object for the specified identifier. If not given the optional identifier argument, the DELETEACL command removes the ACL from the object entirely, causing access to be controlled by a higher-level default ACL. This form of the DELETEACL command is not permitted on the default ACL for a dataset and servers MUST return a BAD. Newman & Myers Standards Track [Page 46] RFC 2244 ACAP November 1997 Example: C: A223 DELETEACL ("/addressbook/~/public") "anyone" S: A223 OK "Deleteacl complete" C: A224 DELETEACL ("/folder/site") S: A224 BAD "Can't delete ACL from dataset" C: A225 DELETEACL ("/addressbook/user/fred" "addressbook.Email" "barney") S: A225 OK "Deleteacl complete" 6.7.3. MYRIGHTS Command Arguments: acl object Data: intermediate responses: MYRIGHTS Result: OK - myrights completed NO - myrights failure: can't get rights BAD - command unknown or arguments invalid The MYRIGHTS command returns the set of rights that the client has to the given dataset or dataset attribute. Example: C: A003 MYRIGHTS ("/folder/site") S: A003 MYRIGHTS "r" S: A003 OK "Myrights complete" 6.7.4. MYRIGHTS Intermediate Response Data: rights The MYRIGHTS response occurs as a result of a MYRIGHTS command. The argument is the set of rights that the client has for the object referred to in the MYRIGHTS command. 6.7.5. LISTRIGHTS Command Arguments: acl object authentication identifier Data: untagged responses: LISTRIGHTS Result: OK - listrights completed NO - listrights failure: can't get rights list BAD - command unknown or arguments invalid Newman & Myers Standards Track [Page 47] RFC 2244 ACAP November 1997 The LISTRIGHTS command takes an object and an identifier and returns information about what rights the current user may revoke or grant to that identifier in the ACL for that object. Example: C: a001 LISTRIGHTS ("/folder/~/") "smith" S: a001 LISTRIGHTS "xra" "w" "i" S: a001 OK Listrights completed C: a005 LISTRIGHTS ("/folder/site/archive/imap") "anyone" S: a005 LISTRIGHTS "" "x" "r" "w" "i" S: a005 OK Listrights completed 6.7.6. LISTRIGHTS Intermediate Response Data: required rights list of optional rights The LISTRIGHTS response occurs as a result of a LISTRIGHTS command. The first argument is a string containing the (possibly empty) set of rights the identifier will always be granted on the dataset or attribute. Following this are zero or more strings each containing a single right which the current user may revoke or grant to the identifier in the dataset or attribute. The same right MUST NOT be listed more than once in the LISTRIGHTS response. 6.8. Quotas The section defines the commands and responses relating to quotas. 6.8.1. GETQUOTA Command Arguments: dataset Data: untagged responses: QUOTA Result: OK - Quota information returned NO - Quota failure: can't access resource limit no resource limit BAD - command unknown or arguments invalid Newman & Myers Standards Track [Page 48] RFC 2244 ACAP November 1997 The GETQUOTA command takes the name of a dataset, and returns in an untagged QUOTA response the name of the dataset, quota limit in bytes that applies to that dataset and the quota usage within that limit. The scope of a quota limit is implementation dependent. Example: C: A043 GETQUOTA "/option/user/fred/common" S: * QUOTA "/option/user/fred/common" 1048576 2475 S: A043 OK "Getquota completed" 6.8.3. QUOTA Untagged Response Data: dataset quota limit in bytes amount of quota limit used extension data The QUOTA untagged response is generated as a result of a GETQUOTA command or MAY be generated by the server in response to a SEARCH or STORE command to warn about high usage of a quota. It includes the name of the applicable dataset, the quota limit in bytes, the quota usage and some optional extension data. Clients MUST tolerate the extension data as its use is reserved for a future extension. 6.9. Extensions In order to simplify the process of extending the protocol, clients MUST tolerate unknown server responses which meet the syntax of response-extend. In addition, clients MUST tolerate unknown server response codes which meet the syntax of resp-code-ext. Availability of new commands MUST be announced via a capability on the initial greeting line and such commands SHOULD meet the syntax of command-extend. Servers MUST respond to unknown commands with a BAD command completion result. Servers MUST skip over non-synchronizing literals contained in an unknown command. This may be done by assuming the unknown command matches the command-extend syntax, or by reading a line at a time and checking for the non-synchronizing literal syntax at the end of the line. 7. Registration Procedures ACAP's usefulness comes from providing a structured storage model for all sorts of configuration data. However, for its potential to be achieved, it is important that the Internet community strives for the following goals: Newman & Myers Standards Track [Page 49] RFC 2244 ACAP November 1997 (1) Standardization. It is very important to standardize dataset classes. The authors hope that ACAP achieves the success that SNMP has seen with the definition of numerous standards track MIBs. (2) Community Review. In the absence of standardization, it is important to get community review on a proposal to improve its engineering quality. Community review is strongly recommended prior to registration. The ACAP implementors mailing list should be used for this purpose. (3) Registration. Registration serves a two-fold purpose. First it prevents use of the same name for different purposes, and second it provides a one-stop list which can be used to locate existing extensions or dataset classes to prevent duplicate work. The following registration templates may be used to register ACAP protocol elements with the Internet Assigned Numbers Authority (IANA). 7.1. ACAP Capabilities New ACAP capabilities MUST be registered prior to use. Careful consideration should be made before extending the protocol, as it can lead to complexity or interoperability problems. Review of proposals on the acap implementors mailing list is strongly encouraged prior to registration. To: iana@iana.org Subject: Registration of ACAP capability Capability name: Capability keyword: Capability arguments: Published Specification(s): (Optional, but strongly encouraged) Person and email address to contact for further information: 7.2. ACAP Response Codes ACAP response codes are registered on a first come, first served basis. Review of proposals on the acap implementors mailing list is strongly encouraged prior to registration. Newman & Myers Standards Track [Page 50] RFC 2244 ACAP November 1997 To: iana@iana.org Subject: Registration of ACAP response code Response Code: Arguments (use ABNF to specify syntax): Purpose: Published Specification(s): (Optional, but strongly encouraged) Person and email address to contact for further information: 7.3. Dataset Classes A dataset class provides a core set of attributes for use in a specified hierarchy. It may also define rules for the dataset hierarchy underneath that class. Dataset class specifications must be standards track or IESG approved experimental RFCs. To: iana@iana.org Subject: Registration of ACAP dataset class Dataset class name/attribute prefix: Purpose: Published Specification(s): (Standards track or IESG approved experimental RFC) Person and email address to contact for further information: 7.4. Vendor Subtree Vendors may reserve a portion of the ACAP namespace for private use. Dataset class names beginning with "vendor.." are reserved for use by that company or product. In addition, all attribute names beginning with "vendor.." are reserved for use by that company or product once registered. Registration is on a first come, first served basis. Whenever possible, private attributes and dataset classes should be avoided in favor of improving interoperable dataset class definitions. Newman & Myers Standards Track [Page 51] RFC 2244 ACAP November 1997 To: iana@iana.org Subject: Registration of ACAP vendor subtree Private Prefix: vendor.. Person and email address to contact for further information: (company names and addresses should be included when appropriate) 8. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [ABNF]. This uses the ABNF core rules as specified in Appendix A of the ABNF specification [ABNF]. Except as noted otherwise, all alphabetic characters are case-insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. The "initial-greeting" rule below defines the initial ACAP greeting from the server. The "command" rule below defines the syntax for commands sent by the client. The "response" rule below defines the syntax for responses sent by the server. ATOM-CHAR = "!" / %x23-27 / %x2A-5B / %x5D-7A / %x7C-7E ;; Any CHAR except ATOM-SPECIALS ATOM-SPECIALS = "(" / ")" / "{" / SP / CTL / QUOTED-SPECIALS CHAR = %x01-7F DIGIT-NZ = %x31-39 ; non-zero digits ("1" - "9") QUOTED-CHAR = SAFE-UTF8-CHAR / "\" QUOTED-SPECIALS QUOTED-SPECIALS = <"> / "\" SAFE-CHAR = %x01-09 / %x0B-0C / %x0E-21 / %x23-5B / %x5D-7F ;; any TEXT-CHAR except QUOTED-SPECIALS SAFE-UTF8-CHAR = SAFE-CHAR / UTF8-2 / UTF8-3 / UTF8-4 / UTF8-5 / UTF8-6 TAG-CHAR = %x21 / %x23-27 / %x2C-5B / %x5D-7A / %x7C-7E ;; Any ATOM-CHAR except "*" or "+" Newman & Myers Standards Track [Page 52] RFC 2244 ACAP November 1997 TEXT-CHAR = %x01-09 / %x0B-0C / %x0E-7F ;; any CHAR except CR and LF TEXT-UTF8-CHAR = SAFE-UTF8-CHAR / QUOTED-SPECIALS UTF8-1 = %x80-BF UTF8-2 = %xC0-DF UTF8-1 UTF8-3 = %xE0-EF 2UTF8-1 UTF8-4 = %xF0-F7 3UTF8-1 UTF8-5 = %xF8-FB 4UTF8-1 UTF8-6 = %xFC-FD 5UTF8-1 UTF8-CHAR = TEXT-UTF8-CHAR / CR / LF acl = "(" [acl-identrights *(SP acl-identrights)] ")" *(SPACE acl-identrights)] ")" acl-identifier = string-utf8 ;; MUST NOT contain HTAB acl-identrights = string-utf8 ;; The identifier followed by a HTAB, ;; followed by the rights. acl-delobject = "(" dataset SP attribute [SP entry-name] ")" acl-object = "(" dataset [SP attribute [SP entry-name]] ")" acl-rights = quoted atom = ALPHA *1023ATOM-CHAR attribute = string-utf8 ;; dot-separated attribute name ;; MUST NOT contain "*" or "%" attribute-store = attribute SP (value-nildef / "(" 1*(metadata-write-q SP value-store) ")") ;; MUST NOT include the same metadata twice auth-type = <"> auth-type-name <"> Newman & Myers Standards Track [Page 53] RFC 2244 ACAP November 1997 auth-type-name = iana-token ;; as defined in SASL [SASL] command = tag SP (command-any / command-auth / command-nonauth) CRLF ;; Modal based on state command-authent = "AUTHENTICATE" SP auth-type [SP string] *(CRLF string) command-any = "NOOP" / command-lang / "LOGOUT" / command-extend command-auth = command-delacl / command-dsince / command-freectx / command-getquota / command-lrights / command-myrights / command-search / command-setacl / command-store ;; only valid in authenticated state command-delacl = "DELETEACL" SP acl-delobject [SP acl-identifier] command-dsince = "DELETEDSINCE" SP dataset SP time command-extend = extend-token [SP extension-data] command-freectx = "FREECONTEXT" SP context command-getquota = "GETQUOTA" SP dataset command-lang = "LANG" *(SP lang-tag) command-lrights = "LISTRIGHTS" SP acl-object command-myrights = "MYRIGHTS" SP acl-object command-nonauth = command-authent ;; only valid in non-authenticated state command-search = "SEARCH" SP (dataset / context) *(SP search-modifier) SP search-criteria ;; MUST NOT include same search-modifier twice command-setacl = "SETACL" SP acl-object SP acl-identifier SP acl-rights command-store = "STORE" SP store-entry-list Newman & Myers Standards Track [Page 54] RFC 2244 ACAP November 1997 comparator = <"> comparator-name <"> comparator-name = ["+" / "-"] iana-token context = string-utf8 ;; MUST NOT begin with slash ("/") dataset = string-utf8 ;; slash-separated dataset name ;; begins with slash entry = entry-name / entry-path entry-name = string-utf8 ;; entry name MUST NOT contain slash ;; MUST NOT begin with "." entry-path = string-utf8 ;; slash-separated path to entry ;; begins with slash entry-relative = string-utf8 ;; potentially relative path to entry extend-token = atom ;; MUST be defined by a standards track or ;; IESG approved experimental protocol extension extension-data = extension-item *(SP extension-item) extension-item = extend-token / string / number / "(" [extension-data] ")" iana-token = atom ;; MUST be registered with IANA initial-greeting = "*" SP "ACAP" *(SP "(" init-capability ")") CRLF init-capability = init-cap-context / init-cap-extend / init-cap-implem / init-cap-sasl init-cap-context = "CONTEXTLIMIT" SP string init-cap-extend = iana-token [SP string-list] init-cap-implem = "IMPLEMENTATION" SP string init-cap-sasl = "SASL" SP string-list Newman & Myers Standards Track [Page 55] RFC 2244 ACAP November 1997 lang-tag = <"> Language-Tag <"> ;; Language-Tag rule is defined in [LANG-TAGS] literal = "{" number [ "+" ] "}" CRLF *OCTET ;; The number represents the number of octets ;; MUST be literal-utf8 except for values literal-utf8 = "{" number [ "+" ] "}" CRLF *UTF8-CHAR ;; The number represents the number of octets ;; not the number of characters metadata = attribute [ "(" metadata-type-list ")" ] ;; attribute MAY end in "*" as wildcard. metadata-list = metadata *(SP metadata) metadata-type = "attribute" / "myrights" / "size" / "count" / metadata-write metadata-type-q = <"> metadata-type <"> metadata-type-list = metadata-type-q *(SP metadata-type-q) metadata-write = "value" / "acl" metadata-write-q = <"> metadata-write <"> nil = "NIL" number = *DIGIT ;; A 32-bit unsigned number. ;; (0 <= n < 4,294,967,296) nz-number = DIGIT-NZ *DIGIT ;; A 32-bit unsigned non-zero number. ;; (0 < n < 4,294,967,296) position = number ;; "0" if context is not enumerated ;; otherwise this is non-zero quota-limit = number quota-usage = number quoted = <"> *QUOTED-CHAR <"> ;; limited to 1024 octets between the <">s Newman & Myers Standards Track [Page 56] RFC 2244 ACAP November 1997 response = response-addto / response-alert / response-bye / response-change / response-cont / response-deleted / response-done / response-entry / response-extend / response-listr / response-lang / response-mtimei / response-mtimeu / response-myright / response-quota / response-refer / response-remove / response-stat response-addto = "*" SP "ADDTO" SP context SP entry-name SP position SP return-data-list response-alert = "*" SP "ALERT" SP resp-body CRLF ;; Client MUST display alert text to user response-bye = "*" SP "BYE" SP resp-body CRLF ;; Server will disconnect condition response-change = "*" SP "CHANGE" SP context SP entry-name SP position SP position SP return-data-list response-cont = "+" SP string response-deleted = tag SP "DELETED" SP entry-name response-done = tag SP resp-cond-state CRLF response-entry = tag SP "ENTRY" SP entry SP return-data-list response-extend = (tag / "*") SP extend-token [SP extension-data] response-lang = "*" SP "LANG" SP lang-tag 1*(SP comparator) response-listr = tag SP "LISTRIGHTS" SP acl-rights *(SP acl-rights) response-mtimei = tag SP "MODTIME" SP time response-mtimeu = "*" SP "MODTIME" SP context SP time response-myright = tag SP "MYRIGHTS" SP acl-rights response-quota = "*" SP "QUOTA" SP dataset SP quota-limit SP quota-usage [SP extension-data] response-refer = tag SP "REFER" SP dataset 1*(SP <"> url-relative <">) Newman & Myers Standards Track [Page 57] RFC 2244 ACAP November 1997 response-remove = "*" SP "REMOVEFROM" SP context SP entry-name SP position response-stat = "*" SP resp-cond-state CRLF resp-body = ["(" resp-code ")" SP] quoted resp-code = "AUTH-TOO-WEAK" / "ENCRYPT-NEEDED" / resp-code-inval / resp-code-mod / resp-code-noexist / resp-code-perm / "QUOTA" / resp-code-refer / resp-code-sasl / resp-code-toomany / "TOOOLD" / "TRANSITION-NEEDED" / "TRYFREECONTEXT" / "TRYLATER" / "WAYTOOMANY" / resp-code-ext resp-code-ext = iana-token [SP extension-data] ;; unknown codes MUST be tolerated by the client resp-code-inval = "INVALID" 1*(SP entry-path SP attribute) resp-code-mod = "MODIFIED" SP entry-path resp-code-noexist = "NOEXIST" SP dataset resp-code-perm = "PERMISSION" SP acl-object resp-code-refer = "REFER" 1*(SP <"> url-relative <">) resp-code-sasl = "SASL" SP string resp-code-toomany = "TOOMANY" SP nz-number resp-cond-state = ("OK" / "NO" / "BAD") SP resp-body ;; Status condition return-attr-list = "(" return-metalist *(SP return-metalist) ")" ;; occurs when "*" in RETURN pattern on SEARCH return-data = return-metadata / return-metalist / return-attr-list return-data-list = return-data *(SP return-data) return-metalist = "(" return-metadata *(SP return-metadata) ")" ;; occurs when multiple metadata items requested return-metadata = nil / string / value-list / acl Newman & Myers Standards Track [Page 58] RFC 2244 ACAP November 1997 searchkey-equal = "EQUAL" SP attribute SP comparator SP value-nil searchkey-comp = "COMPARE" SP attribute SP comparator SP value searchkey-prefix = "PREFIX" SP attribute SP comparator SP value searchkey-range = "RANGE" SP nz-number SP nz-number SP time searchkey-strict = "COMPARESTRICT" SP attribute SP comparator SP value searchkey-substr = "SUBSTRING" SP attribute SP comparator SP value searchmod-depth = "DEPTH" SP number searchmod-hard = "HARDLIMIT" SP nz-number searchmod-limit = "LIMIT" SP number SP number searchmod-make = "MAKECONTEXT" [SP "ENUMERATE"] [SP "NOTIFY"] SP context searchmod-ninh = "NOINHERIT" searchmod-return = "RETURN" SP "(" [metadata-list] ")" searchmod-sort = "SORT" SP "(" sort-list ")" search-criteria = "ALL" / searchkey-equal / searchkey-comp / searchkey-strict / searchkey-range / searchkey-prefix / searchkey-substr / "NOT" SP search-criteria / "OR" SP search-criteria SP search-criteria / "AND" SP search-criteria SP search-criteria search-modifier = searchmod-depth / searchmod-hard / searchmod-limit / searchmod-make / searchmod-ninh / searchmod-return / searchmod-sort sort-list = sort-item *(SP sort-item) sort-item = attribute SP comparator store-entry = "(" entry-path *(SP store-modifier) *(SP attribute-store) ")" ;; MUST NOT include same store-modifier twice ;; MUST NOT include same attribute twice Newman & Myers Standards Track [Page 59] RFC 2244 ACAP November 1997 store-entry-list = store-entry *(SP store-entry) ;; MUST NOT include same entry twice store-modifier = store-mod-unchang / store-mod-nocreate store-mod-nocreate = "NOCREATE" store-mod-unchang = "UNCHANGEDSINCE" SP time string = quoted / literal string-list = string *(SP string) string-utf8 = quoted / literal-utf8 tag = 1*32TAG-CHAR time = <"> time-year time-month time-day time-hour time-minute time-second time-subsecond <"> ;; Timestamp in UTC time-day = 2DIGIT ;; 01-31 time-hour = 2DIGIT ;; 00-23 time-minute = 2DIGIT ;; 00-59 time-month = 2DIGIT ;; 01-12 time-second = 2DIGIT ;; 00-60 time-subsecond = *DIGIT time-year = 4DIGIT value = string value-list = "(" [value *(SP value)] ")" value-nil = value / nil value-nildef = value-nil / "DEFAULT" value-store = value-nildef / value-list / acl url-acap = "acap://" url-server "/" url-enc-entry [url-filter] [url-extension] ;; url-enc-entry interpreted relative to "/" Newman & Myers Standards Track [Page 60] RFC 2244 ACAP November 1997 url-attr-list = url-enc-attr *("&" url-enc-attr) url-auth = ";AUTH=" ("*" / url-enc-auth) url-achar = uchar / "&" / "=" / "~" ;; See RFC 1738 for definition of "uchar" url-char = uchar / "=" / "~" / ":" / "@" / "/" ;; See RFC 1738 for definition of "uchar" url-enc-attr = 1*url-char ;; encoded version of attribute name url-enc-auth = 1*url-achar ;; encoded version of auth-type-name above url-enc-entry = 1*url-char ;; encoded version of entry-relative above url-enc-user = *url-achar ;; encoded version of login userid url-extension = *("?" 1*url-char) url-filter = "?" url-attr-list url-relative = url-acap / [url-enc-entry] [url-filter] ;; url-enc-entry is relative to base URL url-server = [url-enc-user [url-auth] "@"] hostport ;; See RFC 1738 for definition of "hostport" 9. Multi-lingual Considerations The IAB charset workshop [IAB-CHARSET] came to a number of conclusions which influenced the design of ACAP. The decision to use UTF-8 as the character encoding scheme was based on that work. The LANG command to negotiate a language for error messages is also included. Section 3.4.5 of the IAB charset workshop report states that there should be a way to identify the natural language for human readable strings. Several promising proposals have been made for use within ACAP, but no clear consensus on a single method is apparent at this stage. The following rules are likely to permit the addition of multi-lingual support in the future: Newman & Myers Standards Track [Page 61] RFC 2244 ACAP November 1997 (1) A work in progress called Multi-Lingual String Format (MLSF) proposes a layer on top of UTF-8 which uses otherwise illegal UTF-8 sequences to store language tags. In order to permit its addition to a future version of this standard, client-side UTF-8 interpreters MUST be able to silently ignore illegal multi-byte UTF-8 characters, and treat illegal single-byte UTF-8 characters as end of string markers. Servers, for the time being, MUST be able to silently accept illegal UTF-8 characters, except in attribute names and entry names. Clients MUST NOT send illegal UTF-8 characters to the server unless a future standard changes this rule. (2) There is a proposal to add language tags to Unicode. To support this, servers MUST be able to store UTF-8 characters of up to 20 bits of data. (3) The metadata item "language" is reserved for future use. 10. Security Considerations The AUTHENTICATE command uses SASL [SASL] to provide basic authentication, authorization, integrity and privacy services. This is described in section 6.3.1. When the CRAM-MD5 mechanism is used, the security considerations for the CRAM-MD5 SASL mechanism [CRAM-MD5] apply. The CRAM-MD5 mechanism is also susceptible to passive dictionary attacks. This means that if an authentication session is recorded by a passive observer, that observer can try common passwords through the CRAM-MD5 mechanism and see if the results match. This attack is reduced by using hard to guess passwords. Sites are encouraged to educate users and have the password change service test candidate passwords against a dictionary. ACAP implementations of CRAM-MD5 SHOULD permit passwords of at least 64 characters in length. ACAP protocol transactions are susceptible to passive observers or man in the middle attacks which alter the data, unless the optional encryption and integrity services of the AUTHENTICATE command are enabled, or an external security mechanism is used for protection. It may be useful to allow configuration of both clients and servers to refuse to transfer sensitive information in the absence of strong encryption. ACAP access control lists provide fine grained authorization for access to attributes. A number of related security issues are described in section 3.5. ACAP URLs have the same security considerations as IMAP URLs [IMAP-URL]. Newman & Myers Standards Track [Page 62] RFC 2244 ACAP November 1997 ACAP clients are encouraged to consider the security problems involved with a lab computer situation. Specifically, a client cache of ACAP configuration information MUST NOT allow access by an unauthorized user. One way to assure this is for an ACAP client to be able to completely flush any non-public cached configuration data when a user leaves. As laptop computers can be easily stolen and a cache of configuration data may contain sensitive information, a disconnected mode ACAP client may wish to encrypt and password protect cached configuration information. 11. Acknowledgments Many thanks to the follow people who have contributed to ACAP over the past four years: Wallace Colyer, Mark Crispin, Jack DeWinter, Rob Earhart, Ned Freed, Randy Gellens, Terry Gray, J. S. Greenfield, Steve Dorner, Steve Hole, Steve Hubert, Dave Roberts, Bart Schaefer, Matt Wall and other participants of the IETF ACAP working group. 12. Authors' Addresses Chris Newman Innosoft International, Inc. 1050 Lakes Drive West Covina, CA 91790 USA Email: chris.newman@innosoft.com John Gardiner Myers Netscape Communications 501 East Middlefield Road Mail Stop MV-029 Mountain View, CA 94043 Email: jgmyers@netscape.com Newman & Myers Standards Track [Page 63] RFC 2244 ACAP November 1997 Appendices A. References [ABNF] Crocker, Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, Internet Mail Consortium, Demon Internet Ltd, November 1997. [BASIC-URL] Berners-Lee, Masinter, McCahill, "Uniform Resource Locators (URL)", RFC 1738, CERN, Xerox Coproration, University of Minnesota, December 1994. [CHARSET-LANG-POLICY] Alvestrand, "IETF Policy on Character Sets and Languages", work in progress. [CRAM-MD5] Klensin, Catoe, Krumviede, "IMAP/POP AUTHorize Extension for Simple Challenge/Response", RFC 2195, MCI, September 1997. [IAB-CHARSET] Weider, Preston, Simonsen, Alvestrand, Atkinson, Crispin, Svanberg, "The Report of the IAB Character Set Workshop held 29 February - 1 March, 1996", RFC 2130, April 1997. [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, University of Washington, December 1996. [IMAP-ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, Carnegie Mellon, January 1997. [IMAP-URL] Newman, "IMAP URL Scheme", RFC 2192, Innosoft, July 1997. [ISO-10646] ISO/IEC 10646-1:1993(E) "Information Technology-- Universal Multiple-octet Coded Character Set (UCS)." See also amendments 1 through 7, plus editorial corrections. Newman & Myers Standards Track [Page 64] RFC 2244 ACAP November 1997 [ISO-C] "Programming languages -- C", ISO/IEC 9899:1990, International Organization for Standardization. This is effectively the same as ANSI C standard X3.159-1989. [KEYWORDS] Bradner, "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, Harvard University, March 1997. [LANG-TAGS] Alvestrand, H., "Tags for the Identification of Languages", RFC 1766. [REL-URL] Fielding, "Relative Uniform Resource Locators", RFC 1808, UC Irvine, June 1995. [SASL] Myers, J., "Simple Authentication and Security Layer (SASL)", RFC 2222, Netscape Communications, October 1997. [SASL-ANON] Newman, C., "Anonymous SASL Mechanism", RFC 2245, November 1997. [UNICODE-2] The Unicode Consortium, "The Unicode Standard, Version 2.0", Addison-Wesley, 1996. ISBN 0-201-48345-9. [US-ASCII] "USA Standard Code for Information Interchange," X3.4. American National Standards Institute: New York (1968). [UTF8] Yergeau, F. "UTF-8, a transformation format of Unicode and ISO 10646", RFC 2044, Alis Technologies, October 1996. Newman & Myers Standards Track [Page 65] RFC 2244 ACAP November 1997 B. ACAP Keyword Index ACAP (untagged response) ................................... 26 ADDTO (untagged response) .................................. 40 ALERT (untagged response) .................................. 31 ALL (search keyword) ....................................... 36 AND (search keyword) ....................................... 36 AUTH-TOO-WEAK (response code) .............................. 19 AUTHENTICATE (command) ..................................... 31 BAD (response) ............................................. 30 BYE (untagged response) .................................... 30 CHANGE (untagged response) ................................. 41 COMPARE (search keyword) ................................... 36 COMPARESTRICT (search keyword) ............................. 36 CONTEXTLIMIT (ACAP capability) ............................. 27 DELETEACL (command) ........................................ 46 DELETED (intermediate response) ............................ 45 DELETEDSINCE (command) ..................................... 45 DEPTH (search modifier) .................................... 34 ENCRYPT-NEEDED (response code) ............................. 19 ENTRY (intermediate response) .............................. 37 EQUAL (search keyword) ..................................... 37 FREECONTEXT (command) ...................................... 39 GETQUOTA (command) ......................................... 48 HARDLIMIT (search modifier) ................................ 34 IMPLEMENTATION (ACAP capability) ........................... 27 INVALID (response code) .................................... 19 LANG (command) ............................................. 28 LANG (intermediate response) ............................... 28 LIMIT (search modifier) .................................... 34 LISTRIGHTS (command) ....................................... 47 LISTRIGHTS (intermediate response) ......................... 48 LOGOUT (command) ........................................... 29 MAKECONTEXT (search modifier) .............................. 34 MODIFIED (response code) ................................... 19 MODTIME (intermediate response) ............................ 38 MODTIME (untagged response) ................................ 42 MYRIGHTS (command) ......................................... 47 MYRIGHTS (intermediate response) ........................... 47 NO (response) .............................................. 29 NOCREATE (store modifier) .................................. 44 NOEXIST (response code) .................................... 19 NOINHERIT (search modifier) ................................ 35 NOOP (command) ............................................. 27 NOT (search keyword) ....................................... 37 OK (response) .............................................. 29 OR (search keyword) ........................................ 37 PERMISSION (response code) ................................. 19 Newman & Myers Standards Track [Page 66] RFC 2244 ACAP November 1997 PREFIX (search keyword) .................................... 37 QUOTA (response code) ...................................... 19 QUOTA (untagged response) .................................. 49 RANGE (search keyword) ..................................... 37 REFER (intermediate response) .............................. 38 REFER (response code) ...................................... 19 REMOVEFROM (untagged response) ............................. 41 RETURN (search modifier) ................................... 35 SASL (ACAP capability) ..................................... 27 SASL (response code) ....................................... 20 SEARCH (command) ........................................... 33 SETACL (command) ........................................... 46 SORT (search modifier) ..................................... 36 STORE (command) ............................................ 42 SUBSTRING (search keyword) ................................. 37 TOOMANY (response code) .................................... 20 TOOOLD (response code) ..................................... 20 TRANSITION-NEEDED (response code) .......................... 20 TRYFREECONTEXT (response code) ............................. 20 TRYLATER (response code) ................................... 20 UNCHANGEDSINCE (store modifier) ............................ 44 UPDATECONTEXT (command) .................................... 40 WAYTOOMANY (response code) ................................. 20 acl (attribute metadata) ................................... 12 anyone (ACL identifier) .................................... 17 attribute (attribute metadata) ............................. 12 dataset.acl (dataset attribute) ............................ 24 dataset.acl. (dataset attribute) ................ 24 dataset.inherit (dataset attribute) ........................ 24 entry (predefined attribute) ............................... 11 i;ascii-casemap (comparator) ............................... 16 i;ascii-numeric (comparator) ............................... 16 i;octet (comparator) ....................................... 16 modtime (predefined attribute) ............................. 11 myrights (attribute metadata) .............................. 12 size (attribute metadata) .................................. 13 subdataset (predefined attribute) .......................... 11 value (attribute metadata) ................................. 13 Newman & Myers Standards Track [Page 67] RFC 2244 ACAP November 1997 C. Full Copyright Statement Copyright (C) The Internet Society 1997. All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implmentation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Newman & Myers Standards Track [Page 68] ================================================ FILE: docs/rfcs/rfc2342.IMAP4_Namespace.txt ================================================ Network Working Group M. Gahrns Request for Comments: 2342 Microsoft Category: Standards Track C. Newman Innosoft May 1998 IMAP4 Namespace Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (1998). All Rights Reserved. 1. Abstract IMAP4 [RFC-2060] does not define a default server namespace. As a result, two common namespace models have evolved: The "Personal Mailbox" model, in which the default namespace that is presented consists of only the user's personal mailboxes. To access shared mailboxes, the user must use an escape mechanism to reach another namespace. The "Complete Hierarchy" model, in which the default namespace that is presented includes the user's personal mailboxes along with any other mailboxes they have access to. These two models, create difficulties for certain client operations. This document defines a NAMESPACE command that allows a client to discover the prefixes of namespaces used by a server for personal mailboxes, other users' mailboxes, and shared mailboxes. This allows a client to avoid much of the manual user configuration that is now necessary when mixing and matching IMAP4 clients and servers. 2. Conventions used in this document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. If such lines are wrapped without a new "C:" or "S:" label, then the wrapping is for editorial clarity and is not part of the command. Gahrns & Newman Standards Track [Page 1] RFC 2342 IMAP4 Namespace May 1998 Personal Namespace: A namespace that the server considers within the personal scope of the authenticated user on a particular connection. Typically, only the authenticated user has access to mailboxes in their Personal Namespace. It is the part of the namespace that belongs to the user that is allocated for mailboxes. If an INBOX exists for a user, it MUST appear within the user's personal namespace. In the typical case, there SHOULD be only one Personal Namespace on a server. Other Users' Namespace: A namespace that consists of mailboxes from the Personal Namespaces of other users. To access mailboxes in the Other Users' Namespace, the currently authenticated user MUST be explicitly granted access rights. For example, it is common for a manager to grant to their secretary access rights to their mailbox. In the typical case, there SHOULD be only one Other Users' Namespace on a server. Shared Namespace: A namespace that consists of mailboxes that are intended to be shared amongst users and do not exist within a user's Personal Namespace. The namespaces a server uses MAY differ on a per-user basis. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC-2119]. 3. Introduction and Overview Clients often attempt to create mailboxes for such purposes as maintaining a record of sent messages (e.g. "Sent Mail") or temporarily saving messages being composed (e.g. "Drafts"). For these clients to inter-operate correctly with the variety of IMAP4 servers available, the user must enter the prefix of the Personal Namespace used by the server. Using the NAMESPACE command, a client is able to automatically discover this prefix without manual user configuration. In addition, users are often required to manually enter the prefixes of various namespaces in order to view the mailboxes located there. For example, they might be required to enter the prefix of #shared to view the shared mailboxes namespace. The NAMESPACE command allows a client to automatically discover the namespaces that are available on a server. This allows a client to present the available namespaces to the user in what ever manner it deems appropriate. For example, a Gahrns & Newman Standards Track [Page 2] RFC 2342 IMAP4 Namespace May 1998 client could choose to initially display only personal mailboxes, or it may choose to display the complete list of mailboxes available, and initially position the user at the root of their Personal Namespace. A server MAY choose to make available to the NAMESPACE command only a subset of the complete set of namespaces the server supports. To provide the ability to access these namespaces, a client SHOULD allow the user the ability to manually enter a namespace prefix. 4. Requirements IMAP4 servers that support this extension MUST list the keyword NAMESPACE in their CAPABILITY response. The NAMESPACE command is valid in the Authenticated and Selected state. 5. NAMESPACE Command Arguments: none Response: an untagged NAMESPACE response that contains the prefix and hierarchy delimiter to the server's Personal Namespace(s), Other Users' Namespace(s), and Shared Namespace(s) that the server wishes to expose. The response will contain a NIL for any namespace class that is not available. Namespace_Response_Extensions MAY be included in the response. Namespace_Response_Extensions which are not on the IETF standards track, MUST be prefixed with an "X-". Result: OK - Command completed NO - Error: Can't complete command BAD - argument invalid Example 5.1: =========== < A server that supports a single personal namespace. No leading prefix is used on personal mailboxes and "/" is the hierarchy delimiter.> C: A001 NAMESPACE S: * NAMESPACE (("" "/")) NIL NIL S: A001 OK NAMESPACE command completed Gahrns & Newman Standards Track [Page 3] RFC 2342 IMAP4 Namespace May 1998 Example 5.2: =========== < A user logged on anonymously to a server. No personal mailboxes are associated with the anonymous user and the user does not have access to the Other Users' Namespace. No prefix is required to access shared mailboxes and the hierarchy delimiter is "." > C: A001 NAMESPACE S: * NAMESPACE NIL NIL (("" ".")) S: A001 OK NAMESPACE command completed Example 5.3: =========== < A server that contains a Personal Namespace and a single Shared Namespace. > C: A001 NAMESPACE S: * NAMESPACE (("" "/")) NIL (("Public Folders/" "/")) S: A001 OK NAMESPACE command completed Example 5.4: =========== < A server that contains a Personal Namespace, Other Users' Namespace and multiple Shared Namespaces. Note that the hierarchy delimiter used within each namespace can be different. > C: A001 NAMESPACE S: * NAMESPACE (("" "/")) (("~" "/")) (("#shared/" "/") ("#public/" "/")("#ftp/" "/")("#news." ".")) S: A001 OK NAMESPACE command completed The prefix string allows a client to do things such as automatically creating personal mailboxes or LISTing all available mailboxes within a namespace. Example 5.5: =========== < A server that supports only the Personal Namespace, with a leading prefix of INBOX to personal mailboxes and a hierarchy delimiter of "."> C: A001 NAMESPACE S: * NAMESPACE (("INBOX." ".")) NIL NIL S: A001 OK NAMESPACE command completed Gahrns & Newman Standards Track [Page 4] RFC 2342 IMAP4 Namespace May 1998 < Automatically create a mailbox to store sent items.> C: A002 CREATE "INBOX.Sent Mail" S: A002 OK CREATE command completed Although typically a server will support only a single Personal Namespace, and a single Other User's Namespace, circumstances exist where there MAY be multiples of these, and a client MUST be prepared for them. If a client is configured such that it is required to create a certain mailbox, there can be circumstances where it is unclear which Personal Namespaces it should create the mailbox in. In these situations a client SHOULD let the user select which namespaces to create the mailbox in. Example 5.6: =========== < In this example, a server supports 2 Personal Namespaces. In addition to the regular Personal Namespace, the user has an additional personal namespace to allow access to mailboxes in an MH format mailstore. > < The client is configured to save a copy of all mail sent by the user into a mailbox called 'Sent Mail'. Furthermore, after a message is deleted from a mailbox, the client is configured to move that message to a mailbox called 'Deleted Items'.> < Note that this example demonstrates how some extension flags can be passed to further describe the #mh namespace. > C: A001 NAMESPACE S: * NAMESPACE (("" "/")("#mh/" "/" "X-PARAM" ("FLAG1" "FLAG2"))) NIL NIL S: A001 OK NAMESPACE command completed < It is desired to keep only one copy of sent mail. It is unclear which Personal Namespace the client should use to create the 'Sent Mail' mailbox. The user is prompted to select a namespace and only one 'Sent Mail' mailbox is created. > C: A002 CREATE "Sent Mail" S: A002 OK CREATE command completed < The client is designed so that it keeps two 'Deleted Items' mailboxes, one for each namespace. > C: A003 CREATE "Delete Items" S: A003 OK CREATE command completed Gahrns & Newman Standards Track [Page 5] RFC 2342 IMAP4 Namespace May 1998 C: A004 CREATE "#mh/Deleted Items" S: A004 OK CREATE command completed The next level of hierarchy following the Other Users' Namespace prefix SHOULD consist of , where is a user name as per the IMAP4 LOGIN or AUTHENTICATE command. A client can construct a LIST command by appending a "%" to the Other Users' Namespace prefix to discover the Personal Namespaces of other users that are available to the currently authenticated user. In response to such a LIST command, a server SHOULD NOT return user names that have not granted access to their personal mailboxes to the user in question. A server MAY return a LIST response containing only the names of users that have explicitly granted access to the user in question. Alternatively, a server MAY return NO to such a LIST command, requiring that a user name be included with the Other Users' Namespace prefix before listing any other user's mailboxes. Example 5.7: =========== < A server that supports providing a list of other user's mailboxes that are accessible to the currently logged on user. > C: A001 NAMESPACE S: * NAMESPACE (("" "/")) (("Other Users/" "/")) NIL S: A001 OK NAMESPACE command completed C: A002 LIST "" "Other Users/%" S: * LIST () "/" "Other Users/Mike" S: * LIST () "/" "Other Users/Karen" S: * LIST () "/" "Other Users/Matthew" S: * LIST () "/" "Other Users/Tesa" S: A002 OK LIST command completed Example 5.8: =========== < A server that does not support providing a list of other user's mailboxes that are accessible to the currently logged on user. The mailboxes are listable if the client includes the name of the other user with the Other Users' Namespace prefix. > Gahrns & Newman Standards Track [Page 6] RFC 2342 IMAP4 Namespace May 1998 C: A001 NAMESPACE S: * NAMESPACE (("" "/")) (("#Users/" "/")) NIL S: A001 OK NAMESPACE command completed < In this example, the currently logged on user has access to the Personal Namespace of user Mike, but the server chose to suppress this information in the LIST response. However, by appending the user name Mike (received through user input) to the Other Users' Namespace prefix, the client is able to get a listing of the personal mailboxes of user Mike. > C: A002 LIST "" "#Users/%" S: A002 NO The requested item could not be found. C: A003 LIST "" "#Users/Mike/%" S: * LIST () "/" "#Users/Mike/INBOX" S: * LIST () "/" "#Users/Mike/Foo" S: A003 OK LIST command completed. A prefix string might not contain a hierarchy delimiter, because in some cases it is not needed as part of the prefix. Example 5.9: =========== < A server that allows access to the Other Users' Namespace by prefixing the others' mailboxes with a '~' followed by , where is a user name as per the IMAP4 LOGIN or AUTHENTICATE command.> C: A001 NAMESPACE S: * NAMESPACE (("" "/")) (("~" "/")) NIL S: A001 OK NAMESPACE command completed < List the mailboxes for user mark > C: A002 LIST "" "~mark/%" S: * LIST () "/" "~mark/INBOX" S: * LIST () "/" "~mark/foo" S: A002 OK LIST command completed Historical convention has been to start all namespaces with the "#" character. Namespaces that include the "#" character are not IMAP URL [IMAP-URL] friendly requiring the "#" character to be represented as %23 when within URLs. As such, server implementers MAY instead consider using namespace prefixes that do not contain the "#" character. Gahrns & Newman Standards Track [Page 7] RFC 2342 IMAP4 Namespace May 1998 6. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. atom = ; as defined in [RFC-2060] Namespace = nil / "(" 1*( "(" string SP (<"> QUOTED_CHAR <"> / nil) *(Namespace_Response_Extension) ")" ) ")" Namespace_Command = "NAMESPACE" Namespace_Response_Extension = SP string SP "(" string *(SP string) ")" Namespace_Response = "*" SP "NAMESPACE" SP Namespace SP Namespace SP Namespace ; The first Namespace is the Personal Namespace(s) ; The second Namespace is the Other Users' Namespace(s) ; The third Namespace is the Shared Namespace(s) nil = ; as defined in [RFC-2060] QUOTED_CHAR = ; as defined in [RFC-2060] string = ; as defined in [RFC-2060] ; Note that the namespace prefix is to a mailbox and following ; IMAP4 convention, any international string in the NAMESPACE ; response MUST be of modified UTF-7 format as described in ; [RFC-2060]. 7. Security Considerations In response to a LIST command containing an argument of the Other Users' Namespace prefix, a server SHOULD NOT list users that have not granted list access to their personal mailboxes to the currently authenticated user. Providing such a list, could compromise security by potentially disclosing confidential information of who is located on the server, or providing a starting point of a list of user accounts to attack. Gahrns & Newman Standards Track [Page 8] RFC 2342 IMAP4 Namespace May 1998 8. References [RFC-2060], Crispin, M., "Internet Message Access Protocol Version 4rev1", RFC 2060, December 1996. [RFC-2119], Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [IMAP-URL], Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. 9. Acknowledgments Many people have participated in the discussion of IMAP namespaces on the IMAP mailing list. In particular, the authors would like to thank Mark Crispin for many of the concepts relating to the Personal Namespace and accessing the Personal Namespace of other users, Steve Hole for summarizing the two namespace models, John Myers and Jack De Winter for their work in a preceding effort trying to define a standardized personal namespace, and Larry Osterman for his review and collaboration on this document. 11. Authors' Addresses Mike Gahrns Microsoft One Microsoft Way Redmond, WA, 98072, USA Phone: (425) 936-9833 EMail: mikega@microsoft.com Chris Newman Innosoft International, Inc. 1050 East Garvey Ave. South West Covina, CA, 91790, USA EMail: chris.newman@innosoft.com Gahrns & Newman Standards Track [Page 9] RFC 2342 IMAP4 Namespace May 1998 12. Full Copyright Statement Copyright (C) The Internet Society (1998). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Gahrns & Newman Standards Track [Page 10] ================================================ FILE: docs/rfcs/rfc2359.IMAP4_UIDPLUS_extension.txt ================================================ Network Working Group J. Myers Request for Comments: 2359 Netscape Communications Category: Standards Track June 1998 IMAP4 UIDPLUS extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (1998). All Rights Reserved. IESG NOTE The IMAP extension described here assumes a particular means of using IMAP to support disconnected operation. However, this means of supporting disconnected operation is not yet documented. Also, there are multiple theories about how best to do disconnected operation in IMAP, and as yet, there is no consensus on which one should be adopted as a standard. This document is being approved as a Proposed Standard because it does not appear to have technical flaws in itelf. However, approval of this document as a Proposed Standard should not be considered an IETF endorsement of any particular means of doing disconnected operation in IMAP. Table of Contents 1. Abstract .............................................. 2 2. Conventions Used in this Document ..................... 2 3. Introduction and Overview ............................. 2 4. Features .............................................. 2 4.1. UID EXPUNGE Command ................................... 2 4.2. APPENDUID response code ............................... 3 4.3. COPYUID response code ................................. 4 5. Formal Syntax ......................................... 4 6. References ............................................ 4 7. Security Considerations ............................... 5 8. Author's Address ...................................... 5 9. Full Copyright Statement .............................. 6 Myers Standards Track [Page 1] RFC 2359 IMAP4 UIDPLUS extension June 1998 1. Abstract The UIDPLUS extension of the Internet Message Access Protocol [IMAP4] provides a set of features intended to reduce the amount of time and resources used by some client operations. The features in UIDPLUS are primarily intended for disconnected-use clients. 2. Conventions Used in this Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 3. Introduction and Overview The UIDPLUS extension is present in any IMAP4 server implementation which returns "UIDPLUS" as one of the supported capabilities to the CAPABILITY command. The UIDPLUS extension contains one additional command and additional data returned with successful APPEND and COPY commands. Clients that wish to use the new command in UIDPLUS must of course first test for the presence of the extension by issuing a CAPABILITY command. Each of the features in UIDPLUS are optimizations; clients can provide the same functionality, albeit more slowly, by using commands in the base protocol. With each feature, this document recommends a fallback approach to take when the UIDPLUS extension is not supported by the server. 4. Features 4.1. UID EXPUNGE Command Arguments: message set Data: untagged responses: EXPUNGE Result: OK - expunge completed NO - expunge failure (e.g. permission denied) BAD - command unknown or arguments invalid Myers Standards Track [Page 2] RFC 2359 IMAP4 UIDPLUS extension June 1998 The UID EXPUNGE command permanently removes from the currently selected mailbox all messages that both have the \Deleted flag set and have a UID that is included in the specified message set. If a message either does not have the \Deleted flag set or is has a UID that is not included in the specified message set, it is not affected. This command may be used to ensure that a replayed EXPUNGE command does not remove any messages that have been marked as \Deleted between the time that the user requested the expunge operation and the time the server processes the command. If the server does not support the UIDPLUS capability, the client should fall back to using the STORE command to temporarily remove the \Deleted flag from messages it does not want to remove. The client could alternatively fall back to using the EXPUNGE command, risking the unintended removal of some messages. Example: C: A003 UID EXPUNGE 3000:3002 S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: A003 OK UID EXPUNGE completed 4.2. APPENDUID response code Successful APPEND commands return an APPENDUID response code in the tagged OK response. The APPENDUID response code contains as arguments the UIDVALIDITY of the destination mailbox and the UID assigned to the appended message. If the server does not support the UIDPLUS capability, the client can only discover this information by selecting the destination mailbox and issuing FETCH commands. Example: C: A003 APPEND saved-messages (\Seen) {310} C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK [APPENDUID 38505 3955] APPEND completed Myers Standards Track [Page 3] RFC 2359 IMAP4 UIDPLUS extension June 1998 4.3. COPYUID response code Successful COPY and UID COPY commands return a COPYUID response code in the tagged OK response whenever at least one message was copied. The COPYUID response code contains as an argument the UIDVALIDITY of the appended-to mailbox, a message set containing the UIDs of the messages copied to the destination mailbox, in the order they were copied, and a message containing the UIDs assigned to the copied messages, in the order they were assigned. Neither of the message sets may contain extraneous UIDs or the symbol '*'. If the server does not support the UIDPLUS capability, the client can only discover this information by selecting the destination mailbox and issuing FETCH commands. Example: C: A003 COPY 2:4 MEETING S: A003 OK [COPYUID 38505 304,319:320 3956:3958] Done C: A003 UID COPY 305:310 MEETING S: A003 OK Done 5. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [RFC-822] as modified by [IMAP4]. Non-terminals referenced but not defined below are as defined by [IMAP4]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. resp_code_apnd ::= "APPENDUID" SPACE nz_number SPACE uniqueid resp_code_copy ::= "COPYUID" SPACE nz_number SPACE set SPACE set uid_expunge ::= "UID" SPACE "EXPUNGE" SPACE set 6. References [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822, August 1982. Myers Standards Track [Page 4] RFC 2359 IMAP4 UIDPLUS extension June 1998 7. Security Considerations There are no known security issues with this extension. 8. Author's Address John Gardiner Myers Netscape Communications 501 East Middlefield Road Mail Stop MV-029 Mountain View, CA 94043 EMail: jgmyers@netscape.com Myers Standards Track [Page 5] RFC 2359 IMAP4 UIDPLUS extension June 1998 9. Full Copyright Statement Copyright (C) The Internet Society (1998). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Myers Standards Track [Page 6] ================================================ FILE: docs/rfcs/rfc2595.TLS_with_IMAP-POP3_and_ACAP.txt ================================================ Network Working Group C. Newman Request for Comments: 2595 Innosoft Category: Standards Track June 1999 Using TLS with IMAP, POP3 and ACAP Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (1999). All Rights Reserved. 1. Motivation The TLS protocol (formerly known as SSL) provides a way to secure an application protocol from tampering and eavesdropping. The option of using such security is desirable for IMAP, POP and ACAP due to common connection eavesdropping and hijacking attacks [AUTH]. Although advanced SASL authentication mechanisms can provide a lightweight version of this service, TLS is complimentary to simple authentication-only SASL mechanisms or deployed clear-text password login commands. Many sites have a high investment in authentication infrastructure (e.g., a large database of a one-way-function applied to user passwords), so a privacy layer which is not tightly bound to user authentication can protect against network eavesdropping attacks without requiring a new authentication infrastructure and/or forcing all users to change their password. Recognizing that such sites will desire simple password authentication in combination with TLS encryption, this specification defines the PLAIN SASL mechanism for use with protocols which lack a simple password authentication command such as ACAP and SMTP. (Note there is a separate RFC for the STARTTLS command in SMTP [SMTPTLS].) There is a strong desire in the IETF to eliminate the transmission of clear-text passwords over unencrypted channels. While SASL can be used for this purpose, TLS provides an additional tool with different deployability characteristics. A server supporting both TLS with Newman Standards Track [Page 1] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 simple passwords and a challenge/response SASL mechanism is likely to interoperate with a wide variety of clients without resorting to unencrypted clear-text passwords. The STARTTLS command rectifies a number of the problems with using a separate port for a "secure" protocol variant. Some of these are mentioned in section 7. 1.1. Conventions Used in this Document The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to be interpreted as described in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. Terms related to authentication are defined in "On Internet Authentication" [AUTH]. Formal syntax is defined using ABNF [ABNF]. In examples, "C:" and "S:" indicate lines sent by the client and server respectively. 2. Basic Interoperability and Security Requirements The following requirements apply to all implementations of the STARTTLS extension for IMAP, POP3 and ACAP. 2.1. Cipher Suite Requirements Implementation of the TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher suite is REQUIRED. This is important as it assures that any two compliant implementations can be configured to interoperate. All other cipher suites are OPTIONAL. 2.2. Privacy Operational Mode Security Requirements Both clients and servers SHOULD have a privacy operational mode which refuses authentication unless successful activation of an encryption layer (such as that provided by TLS) occurs prior to or at the time of authentication and which will terminate the connection if that encryption layer is deactivated. Implementations are encouraged to have flexability with respect to the minimal encryption strength or cipher suites permitted. A minimalist approach to this recommendation would be an operational mode where the TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA cipher suite is mandatory prior to permitting authentication. Newman Standards Track [Page 2] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 Clients MAY have an operational mode which uses encryption only when it is advertised by the server, but authentication continues regardless. For backwards compatibility, servers SHOULD have an operational mode where only the authentication mechanisms required by the relevant base protocol specification are needed to successfully authenticate. 2.3. Clear-Text Password Requirements Clients and servers which implement STARTTLS MUST be configurable to refuse all clear-text login commands or mechanisms (including both standards-track and nonstandard mechanisms) unless an encryption layer of adequate strength is active. Servers which allow unencrypted clear-text logins SHOULD be configurable to refuse clear-text logins both for the entire server, and on a per-user basis. 2.4. Server Identity Check During the TLS negotiation, the client MUST check its understanding of the server hostname against the server's identity as presented in the server Certificate message, in order to prevent man-in-the-middle attacks. Matching is performed according to these rules: - The client MUST use the server hostname it used to open the connection as the value to compare against the server name as expressed in the server certificate. The client MUST NOT use any form of the server hostname derived from an insecure remote source (e.g., insecure DNS lookup). CNAME canonicalization is not done. - If a subjectAltName extension of type dNSName is present in the certificate, it SHOULD be used as the source of the server's identity. - Matching is case-insensitive. - A "*" wildcard character MAY be used as the left-most name component in the certificate. For example, *.example.com would match a.example.com, foo.example.com, etc. but would not match example.com. - If the certificate contains multiple names (e.g. more than one dNSName field), then a match with any one of the fields is considered acceptable. If the match fails, the client SHOULD either ask for explicit user confirmation, or terminate the connection and indicate the server's identity is suspect. Newman Standards Track [Page 3] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 2.5. TLS Security Policy Check Both the client and server MUST check the result of the STARTTLS command and subsequent TLS negotiation to see whether acceptable authentication or privacy was achieved. Ignoring this step completely invalidates using TLS for security. The decision about whether acceptable authentication or privacy was achieved is made locally, is implementation-dependent, and is beyond the scope of this document. 3. IMAP STARTTLS extension When the TLS extension is present in IMAP, "STARTTLS" is listed as a capability in response to the CAPABILITY command. This extension adds a single command, "STARTTLS" to the IMAP protocol which is used to begin a TLS negotiation. 3.1. STARTTLS Command Arguments: none Responses: no specific responses for this command Result: OK - begin TLS negotiation BAD - command unknown or arguments invalid A TLS negotiation begins immediately after the CRLF at the end of the tagged OK response from the server. Once a client issues a STARTTLS command, it MUST NOT issue further commands until a server response is seen and the TLS negotiation is complete. The STARTTLS command is only valid in non-authenticated state. The server remains in non-authenticated state, even if client credentials are supplied during the TLS negotiation. The SASL [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS client credentials are successfully exchanged, but servers supporting the STARTTLS command are not required to support the EXTERNAL mechanism. Once TLS has been started, the client MUST discard cached information about server capabilities and SHOULD re-issue the CAPABILITY command. This is necessary to protect against man-in-the-middle attacks which alter the capabilities list prior to STARTTLS. The server MAY advertise different capabilities after STARTTLS. The formal syntax for IMAP is amended as follows: Newman Standards Track [Page 4] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 command_any =/ "STARTTLS" Example: C: a001 CAPABILITY S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED S: a001 OK CAPABILITY completed C: a002 STARTTLS S: a002 OK Begin TLS negotiation now C: a003 CAPABILITY S: * CAPABILITY IMAP4rev1 AUTH=EXTERNAL S: a003 OK CAPABILITY completed C: a004 LOGIN joe password S: a004 OK LOGIN completed 3.2. IMAP LOGINDISABLED capability The current IMAP protocol specification (RFC 2060) requires the implementation of the LOGIN command which uses clear-text passwords. Many sites may choose to disable this command unless encryption is active for security reasons. An IMAP server MAY advertise that the LOGIN command is disabled by including the LOGINDISABLED capability in the capability response. Such a server will respond with a tagged "NO" response to any attempt to use the LOGIN command. An IMAP server which implements STARTTLS MUST implement support for the LOGINDISABLED capability on unencrypted connections. An IMAP client which complies with this specification MUST NOT issue the LOGIN command if this capability is present. This capability is useful to prevent clients compliant with this specification from sending an unencrypted password in an environment subject to passive attacks. It has no impact on an environment subject to active attacks as a man-in-the-middle attacker can remove this capability. Therefore this does not relieve clients of the need to follow the privacy mode recommendation in section 2.2. Servers advertising this capability will fail to interoperate with many existing compliant IMAP clients and will be unable to prevent those clients from disclosing the user's password. 4. POP3 STARTTLS extension The POP3 STARTTLS extension adds the STLS command to POP3 servers. If this is implemented, the POP3 extension mechanism [POP3EXT] MUST also be implemented to avoid the need for client probing of multiple commands. The capability name "STLS" indicates this command is present and permitted in the current state. Newman Standards Track [Page 5] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 STLS Arguments: none Restrictions: Only permitted in AUTHORIZATION state. Discussion: A TLS negotiation begins immediately after the CRLF at the end of the +OK response from the server. A -ERR response MAY result if a security layer is already active. Once a client issues a STLS command, it MUST NOT issue further commands until a server response is seen and the TLS negotiation is complete. The STLS command is only permitted in AUTHORIZATION state and the server remains in AUTHORIZATION state, even if client credentials are supplied during the TLS negotiation. The AUTH command [POP-AUTH] with the EXTERNAL mechanism [SASL] MAY be used to authenticate once TLS client credentials are successfully exchanged, but servers supporting the STLS command are not required to support the EXTERNAL mechanism. Once TLS has been started, the client MUST discard cached information about server capabilities and SHOULD re-issue the CAPA command. This is necessary to protect against man-in-the-middle attacks which alter the capabilities list prior to STLS. The server MAY advertise different capabilities after STLS. Possible Responses: +OK -ERR Examples: C: STLS S: +OK Begin TLS negotiation ... C: STLS S: -ERR Command not permitted when TLS active Newman Standards Track [Page 6] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 5. ACAP STARTTLS extension When the TLS extension is present in ACAP, "STARTTLS" is listed as a capability in the ACAP greeting. No arguments to this capability are defined at this time. This extension adds a single command, "STARTTLS" to the ACAP protocol which is used to begin a TLS negotiation. 5.1. STARTTLS Command Arguments: none Responses: no specific responses for this command Result: OK - begin TLS negotiation BAD - command unknown or arguments invalid A TLS negotiation begins immediately after the CRLF at the end of the tagged OK response from the server. Once a client issues a STARTTLS command, it MUST NOT issue further commands until a server response is seen and the TLS negotiation is complete. The STARTTLS command is only valid in non-authenticated state. The server remains in non-authenticated state, even if client credentials are supplied during the TLS negotiation. The SASL [SASL] EXTERNAL mechanism MAY be used to authenticate once TLS client credentials are successfully exchanged, but servers supporting the STARTTLS command are not required to support the EXTERNAL mechanism. After the TLS layer is established, the server MUST re-issue an untagged ACAP greeting. This is necessary to protect against man-in-the-middle attacks which alter the capabilities list prior to STARTTLS. The client MUST discard cached capability information and replace it with the information from the new ACAP greeting. The server MAY advertise different capabilities after STARTTLS. The formal syntax for ACAP is amended as follows: command_any =/ "STARTTLS" Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) C: a002 STARTTLS S: a002 OK "Begin TLS negotiation now" S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") Newman Standards Track [Page 7] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 6. PLAIN SASL mechanism Clear-text passwords are simple, interoperate with almost all existing operating system authentication databases, and are useful for a smooth transition to a more secure password-based authentication mechanism. The drawback is that they are unacceptable for use over an unencrypted network connection. This defines the "PLAIN" SASL mechanism for use with ACAP and other protocols with no clear-text login command. The PLAIN SASL mechanism MUST NOT be advertised or used unless a strong encryption layer (such as the provided by TLS) is active or backwards compatibility dictates otherwise. The mechanism consists of a single message from the client to the server. The client sends the authorization identity (identity to login as), followed by a US-ASCII NUL character, followed by the authentication identity (identity whose password will be used), followed by a US-ASCII NUL character, followed by the clear-text password. The client may leave the authorization identity empty to indicate that it is the same as the authentication identity. The server will verify the authentication identity and password with the system authentication database and verify that the authentication credentials permit the client to login as the authorization identity. If both steps succeed, the user is logged in. The server MAY also use the password to initialize any new authentication database, such as one suitable for CRAM-MD5 [CRAM-MD5]. Non-US-ASCII characters are permitted as long as they are represented in UTF-8 [UTF-8]. Use of non-visible characters or characters which a user may be unable to enter on some keyboards is discouraged. The formal grammar for the client message using Augmented BNF [ABNF] follows. message = [authorize-id] NUL authenticate-id NUL password authenticate-id = 1*UTF8-SAFE ; MUST accept up to 255 octets authorize-id = 1*UTF8-SAFE ; MUST accept up to 255 octets password = 1*UTF8-SAFE ; MUST accept up to 255 octets NUL = %x00 UTF8-SAFE = %x01-09 / %x0B-0C / %x0E-7F / UTF8-2 / UTF8-3 / UTF8-4 / UTF8-5 / UTF8-6 UTF8-1 = %x80-BF UTF8-2 = %xC0-DF UTF8-1 UTF8-3 = %xE0-EF 2UTF8-1 Newman Standards Track [Page 8] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 UTF8-4 = %xF0-F7 3UTF8-1 UTF8-5 = %xF8-FB 4UTF8-1 UTF8-6 = %xFC-FD 5UTF8-1 Here is an example of how this might be used to initialize a CRAM-MD5 authentication database for ACAP: Example: S: * ACAP (SASL "CRAM-MD5") (STARTTLS) C: a001 AUTHENTICATE "CRAM-MD5" S: + "<1896.697170952@postoffice.reston.mci.net>" C: "tim b913a602c7eda7a495b4e6e7334d3890" S: a001 NO (TRANSITION-NEEDED) "Please change your password, or use TLS to login" C: a002 STARTTLS S: a002 OK "Begin TLS negotiation now" S: * ACAP (SASL "CRAM-MD5" "PLAIN" "EXTERNAL") C: a003 AUTHENTICATE "PLAIN" {21+} C: timtanstaaftanstaaf S: a003 OK CRAM-MD5 password initialized Note: In this example, represents a single ASCII NUL octet. 7. imaps and pop3s ports Separate "imaps" and "pop3s" ports were registered for use with SSL. Use of these ports is discouraged in favor of the STARTTLS or STLS commands. A number of problems have been observed with separate ports for "secure" variants of protocols. This is an attempt to enumerate some of those problems. - Separate ports lead to a separate URL scheme which intrudes into the user interface in inappropriate ways. For example, many web pages use language like "click here if your browser supports SSL." This is a decision the browser is often more capable of making than the user. - Separate ports imply a model of either "secure" or "not secure." This can be misleading in a number of ways. First, the "secure" port may not in fact be acceptably secure as an export-crippled cipher suite might be in use. This can mislead users into a false sense of security. Second, the normal port might in fact be secured by using a SASL mechanism which includes a security layer. Thus the separate port distinction makes the complex topic of security policy even more confusing. One common result of this confusion is that firewall administrators are often misled into Newman Standards Track [Page 9] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 permitting the "secure" port and blocking the standard port. This could be a poor choice given the common use of SSL with a 40-bit key encryption layer and plain-text password authentication is less secure than strong SASL mechanisms such as GSSAPI with Kerberos 5. - Use of separate ports for SSL has caused clients to implement only two security policies: use SSL or don't use SSL. The desirable security policy "use TLS when available" would be cumbersome with the separate port model, but is simple with STARTTLS. - Port numbers are a limited resource. While they are not yet in short supply, it is unwise to set a precedent that could double (or worse) the speed of their consumption. 8. IANA Considerations This constitutes registration of the "STARTTLS" and "LOGINDISABLED" IMAP capabilities as required by section 7.2.1 of RFC 2060 [IMAP]. The registration for the POP3 "STLS" capability follows: CAPA tag: STLS Arguments: none Added commands: STLS Standard commands affected: May enable USER/PASS as a side-effect. CAPA command SHOULD be re-issued after successful completion. Announced states/Valid states: AUTHORIZATION state only. Specification reference: this memo The registration for the ACAP "STARTTLS" capability follows: Capability name: STARTTLS Capability keyword: STARTTLS Capability arguments: none Published Specification(s): this memo Person and email address for further information: see author's address section below The registration for the PLAIN SASL mechanism follows: SASL mechanism name: PLAIN Security Considerations: See section 9 of this memo Published specification: this memo Person & email address to contact for further information: see author's address section below Intended usage: COMMON Author/Change controller: see author's address section below Newman Standards Track [Page 10] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 9. Security Considerations TLS only provides protection for data sent over a network connection. Messages transferred over IMAP or POP3 are still available to server administrators and usually subject to eavesdropping, tampering and forgery when transmitted through SMTP or NNTP. TLS is no substitute for an end-to-end message security mechanism using MIME security multiparts [MIME-SEC]. A man-in-the-middle attacker can remove STARTTLS from the capability list or generate a failure response to the STARTTLS command. In order to detect such an attack, clients SHOULD warn the user when session privacy is not active and/or be configurable to refuse to proceed without an acceptable level of security. A man-in-the-middle attacker can always cause a down-negotiation to the weakest authentication mechanism or cipher suite available. For this reason, implementations SHOULD be configurable to refuse weak mechanisms or cipher suites. Any protocol interactions prior to the TLS handshake are performed in the clear and can be modified by a man-in-the-middle attacker. For this reason, clients MUST discard cached information about server capabilities advertised prior to the start of the TLS handshake. Clients are encouraged to clearly indicate when the level of encryption active is known to be vulnerable to attack using modern hardware (such as encryption keys with 56 bits of entropy or less). The LOGINDISABLED IMAP capability (discussed in section 3.2) only reduces the potential for passive attacks, it provides no protection against active attacks. The responsibility remains with the client to avoid sending a password over a vulnerable channel. The PLAIN mechanism relies on the TLS encryption layer for security. When used without TLS, it is vulnerable to a common network eavesdropping attack. Therefore PLAIN MUST NOT be advertised or used unless a suitable TLS encryption layer is active or backwards compatibility dictates otherwise. When the PLAIN mechanism is used, the server gains the ability to impersonate the user to all services with the same password regardless of any encryption provided by TLS or other network privacy mechanisms. While many other authentication mechanisms have similar weaknesses, stronger SASL mechanisms such as Kerberos address this issue. Clients are encouraged to have an operational mode where all mechanisms which are likely to reveal the user's password to the server are disabled. Newman Standards Track [Page 11] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 The security considerations for TLS apply to STARTTLS and the security considerations for SASL apply to the PLAIN mechanism. Additional security requirements are discussed in section 2. 10. References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [AUTH] Haller, N. and R. Atkinson, "On Internet Authentication", RFC 1704, October 1994. [CRAM-MD5] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP AUTHorize Extension for Simple Challenge/Response", RFC 2195, September 1997. [IMAP] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MIME-SEC] Galvin, J., Murphy, S., Crocker, S. and N. Freed, "Security Multiparts for MIME: Multipart/Signed and Multipart/Encrypted", RFC 1847, October 1995. [POP3] Myers, J. and M. Rose, "Post Office Protocol - Version 3", STD 53, RFC 1939, May 1996. [POP3EXT] Gellens, R., Newman, C. and L. Lundblade, "POP3 Extension Mechanism", RFC 2449, November 1998. [POP-AUTH] Myers, J., "POP3 AUTHentication command", RFC 1734, December 1994. [SASL] Myers, J., "Simple Authentication and Security Layer (SASL)", RFC 2222, October 1997. [SMTPTLS] Hoffman, P., "SMTP Service Extension for Secure SMTP over TLS", RFC 2487, January 1999. [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", RFC 2246, January 1999. Newman Standards Track [Page 12] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO 10646", RFC 2279, January 1998. 11. Author's Address Chris Newman Innosoft International, Inc. 1050 Lakes Drive West Covina, CA 91790 USA EMail: chris.newman@innosoft.com A. Appendix -- Compliance Checklist An implementation is not compliant if it fails to satisfy one or more of the MUST requirements for the protocols it implements. An implementation that satisfies all the MUST and all the SHOULD requirements for its protocols is said to be "unconditionally compliant"; one that satisfies all the MUST requirements but not all the SHOULD requirements for its protocols is said to be "conditionally compliant". Rules Section ----- ------- Mandatory-to-implement Cipher Suite 2.1 SHOULD have mode where encryption required 2.2 server SHOULD have mode where TLS not required 2.2 MUST be configurable to refuse all clear-text login commands or mechanisms 2.3 server SHOULD be configurable to refuse clear-text login commands on entire server and on per-user basis 2.3 client MUST check server identity 2.4 client MUST use hostname used to open connection 2.4 client MUST NOT use hostname from insecure remote lookup 2.4 client SHOULD support subjectAltName of dNSName type 2.4 client SHOULD ask for confirmation or terminate on fail 2.4 MUST check result of STARTTLS for acceptable privacy 2.5 client MUST NOT issue commands after STARTTLS until server response and negotiation done 3.1,4,5.1 client MUST discard cached information 3.1,4,5.1,9 client SHOULD re-issue CAPABILITY/CAPA command 3.1,4 IMAP server with STARTTLS MUST implement LOGINDISABLED 3.2 IMAP client MUST NOT issue LOGIN if LOGINDISABLED 3.2 POP server MUST implement POP3 extensions 4 ACAP server MUST re-issue ACAP greeting 5.1 Newman Standards Track [Page 13] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 client SHOULD warn when session privacy not active and/or refuse to proceed without acceptable security level 9 SHOULD be configurable to refuse weak mechanisms or cipher suites 9 The PLAIN mechanism is an optional part of this specification. However if it is implemented the following rules apply: Rules Section ----- ------- MUST NOT use PLAIN unless strong encryption active or backwards compatibility dictates otherwise 6,9 MUST use UTF-8 encoding for characters in PLAIN 6 Newman Standards Track [Page 14] RFC 2595 Using TLS with IMAP, POP3 and ACAP June 1999 Full Copyright Statement Copyright (C) The Internet Society (1999). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Newman Standards Track [Page 15] ================================================ FILE: docs/rfcs/rfc2683.IMAP4_Implementation_recommendations.txt ================================================ Network Working Group B. Leiba Request for Comments: 2683 IBM T.J. Watson Research Center Category: Informational September 1999 IMAP4 Implementation Recommendations Status of this Memo This memo provides information for the Internet community. It does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (1999). All Rights Reserved. 1. Abstract The IMAP4 specification [RFC-2060] describes a rich protocol for use in building clients and servers for storage, retrieval, and manipulation of electronic mail. Because the protocol is so rich and has so many implementation choices, there are often trade-offs that must be made and issues that must be considered when designing such clients and servers. This document attempts to outline these issues and to make recommendations in order to make the end products as interoperable as possible. 2. Conventions used in this document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The words "must", "must not", "should", "should not", and "may" are used with specific meaning in this document; since their meaning is somewhat different from that specified in RFC 2119, we do not put them in all caps here. Their meaning is as follows: must -- This word means that the action described is necessary to ensure interoperability. The recommendation should not be ignored. must not -- This phrase means that the action described will be almost certain to hurt interoperability. The recommendation should not be ignored. Leiba Informational [Page 1] RFC 2683 IMAP4 Implementation Recommendations September 1999 should -- This word means that the action described is strongly recommended and will enhance interoperability or usability. The recommendation should not be ignored without careful consideration. should not -- This phrase means that the action described is strongly recommended against, and might hurt interoperability or usability. The recommendation should not be ignored without careful consideration. may -- This word means that the action described is an acceptable implementation choice. No specific recommendation is implied; this word is used to point out a choice that might not be obvious, or to let implementors know what choices have been made by existing implementations. 3. Interoperability Issues and Recommendations 3.1. Accessibility This section describes the issues related to access to servers and server resources. Concerns here include data sharing and maintenance of client/server connections. 3.1.1. Multiple Accesses of the Same Mailbox One strong point of IMAP4 is that, unlike POP3, it allows for multiple simultaneous access to a single mailbox. A user can, thus, read mail from a client at home while the client in the office is still connected; or the help desk staff can all work out of the same inbox, all seeing the same pool of questions. An important point about this capability, though is that NO SERVER IS GUARANTEED TO SUPPORT THIS. If you are selecting an IMAP server and this facility is important to you, be sure that the server you choose to install, in the configuration you choose to use, supports it. If you are designing a client, you must not assume that you can access the same mailbox more than once at a time. That means 1. you must handle gracefully the failure of a SELECT command if the server refuses the second SELECT, 2. you must handle reasonably the severing of your connection (see "Severed Connections", below) if the server chooses to allow the second SELECT by forcing the first off, 3. you must avoid making multiple connections to the same mailbox in your own client (for load balancing or other such reasons), and 4. you must avoid using the STATUS command on a mailbox that you have selected (with some server implementations the STATUS command has the same problems with multiple access as do the SELECT and Leiba Informational [Page 2] RFC 2683 IMAP4 Implementation Recommendations September 1999 EXAMINE commands). A further note about STATUS: The STATUS command is sometimes used to check a non-selected mailbox for new mail. This mechanism must not be used to check for new mail in the selected mailbox; section 5.2 of [RFC-2060] specifically forbids this in its last paragraph. Further, since STATUS takes a mailbox name it is an independent operation, not operating on the selected mailbox. Because of this, the information it returns is not necessarily in synchronization with the selected mailbox state. 3.1.2. Severed Connections The client/server connection may be severed for one of three reasons: the client severs the connection, the server severs the connection, or the connection is severed by outside forces beyond the control of the client and the server (a telephone line drops, for example). Clients and servers must both deal with these situations. When the client wants to sever a connection, it's usually because it has finished the work it needed to do on that connection. The client should send a LOGOUT command, wait for the tagged response, and then close the socket. But note that, while this is what's intended in the protocol design, there isn't universal agreement here. Some contend that sending the LOGOUT and waiting for the two responses (untagged BYE and tagged OK) is wasteful and unnecessary, and that the client can simply close the socket. The server should interpret the closed socket as a log out by the client. The counterargument is that it's useful from the standpoint of cleanup, problem determination, and the like, to have an explicit client log out, because otherwise there is no way for the server to tell the difference between "closed socket because of log out" and "closed socket because communication was disrupted". If there is a client/server interaction problem, a client which routinely terminates a session by breaking the connection without a LOGOUT will make it much more difficult to determine the problem. Because of this disagreement, server designers must be aware that some clients might close the socket without sending a LOGOUT. In any case, whether or not a LOGOUT was sent, the server should not implicitly expunge any messages from the selected mailbox. If a client wants the server to do so, it must send a CLOSE or EXPUNGE command explicitly. When the server wants to sever a connection it's usually due to an inactivity timeout or is because a situation has arisen that has changed the state of the mail store in a way that the server can not communicate to the client. The server should send an untagged BYE Leiba Informational [Page 3] RFC 2683 IMAP4 Implementation Recommendations September 1999 response to the client and then close the socket. Sending an untagged BYE response before severing allows the server to send a human-readable explanation of the problem to the client, which the client may then log, display to the user, or both (see section 7.1.5 of [RFC-2060]). Regarding inactivity timeouts, there is some controversy. Unlike POP, for which the design is for a client to connect, retrieve mail, and log out, IMAP's design encourages long-lived (and mostly inactive) client/server sessions. As the number of users grows, this can use up a lot of server resources, especially with clients that are designed to maintain sessions for mailboxes that the user has finished accessing. To alleviate this, a server may implement an inactivity timeout, unilaterally closing a session (after first sending an untagged BYE, as noted above). Some server operators have reported dramatic improvements in server performance after doing this. As specified in [RFC-2060], if such a timeout is done it must not be until at least 30 minutes of inactivity. The reason for this specification is to prevent clients from sending commands (such as NOOP) to the server at frequent intervals simply to avert a too-early timeout. If the client knows that the server may not time out the session for at least 30 minutes, then the client need not poll at intervals more frequent than, say, 25 minutes. 3.2. Scaling IMAP4 has many features that allow for scalability, as mail stores become larger and more numerous. Large numbers of users, mailboxes, and messages, and very large messages require thought to handle efficiently. This document will not address the administrative issues involved in large numbers of users, but we will look at the other items. 3.2.1. Flood Control There are three situations when a client can make a request that will result in a very large response - too large for the client reasonably to deal with: there are a great many mailboxes available, there are a great many messages in the selected mailbox, or there is a very large message part. The danger here is that the end user will be stuck waiting while the server sends (and the client processes) an enormous response. In all of these cases there are things a client can do to reduce that danger. There is also the case where a client can flood a server, by sending an arbitratily long command. We'll discuss that issue, too, in this section. Leiba Informational [Page 4] RFC 2683 IMAP4 Implementation Recommendations September 1999 3.2.1.1. Listing Mailboxes Some servers present Usenet newsgroups to IMAP users. Newsgroups, and other such hierarchical mailbox structures, can be very numerous but may have only a few entries at the top level of hierarchy. Also, some servers are built against mail stores that can, unbeknownst to the server, have circular hierarchies - that is, it's possible for "a/b/c/d" to resolve to the same file structure as "a", which would then mean that "a/b/c/d/b" is the same as "a/b", and the hierarchy will never end. The LIST response in this case will be unlimited. Clients that will have trouble with this are those that use C: 001 LIST "" * to determine the mailbox list. Because of this, clients should not use an unqualified "*" that way in the LIST command. A safer approach is to list each level of hierarchy individually, allowing the user to traverse the tree one limb at a time, thus: C: 001 LIST "" % S: * LIST () "/" Banana S: * LIST ...etc... S: 001 OK done and then C: 002 LIST "" Banana/% S: * LIST () "/" Banana/Apple S: * LIST ...etc... S: 002 OK done Using this technique the client's user interface can give the user full flexibility without choking on the voluminous reply to "LIST *". Of course, it is still possible that the reply to C: 005 LIST "" alt.fan.celebrity.% may be thousands of entries long, and there is, unfortunately, nothing the client can do to protect itself from that. This has not yet been a notable problem. Servers that may export circular hierarchies (any server that directly presents a UNIX file system, for instance) should limit the hierarchy depth to prevent unlimited LIST responses. A suggested depth limit is 20 hierarchy levels. Leiba Informational [Page 5] RFC 2683 IMAP4 Implementation Recommendations September 1999 3.2.1.2. Fetching the List of Messages When a client selects a mailbox, it is given a count, in the untagged EXISTS response, of the messages in the mailbox. This number can be very large. In such a case it might be unwise to use C: 004 FETCH 1:* ALL to populate the user's view of the mailbox. One good method to avoid problems with this is to batch the requests, thus: C: 004 FETCH 1:50 ALL S: * 1 FETCH ...etc... S: 004 OK done C: 005 FETCH 51:100 ALL S: * 51 FETCH ...etc... S: 005 OK done C: 006 FETCH 101:150 ALL ...etc... Using this method, another command, such as "FETCH 6 BODY[1]" can be inserted as necessary, and the client will not have its access to the server blocked by a storm of FETCH replies. (Such a method could be reversed to fetch the LAST 50 messages first, then the 50 prior to that, and so on.) As a smart extension of this, a well designed client, prepared for very large mailboxes, will not automatically fetch data for all messages AT ALL. Rather, the client will populate the user's view only as the user sees it, possibly pre-fetching selected information, and only fetching other information as the user scrolls to it. For example, to select only those messages beginning with the first unseen one: C: 003 SELECT INBOX S: * 10000 EXISTS S: * 80 RECENT S: * FLAGS (\Answered \Flagged \Deleted \Draft \Seen) S: * OK [UIDVALIDITY 824708485] UID validity status S: * OK [UNSEEN 9921] First unseen message S: 003 OK [READ-WRITE] SELECT completed C: 004 FETCH 9921:* ALL ... etc... If the server does not return an OK [UNSEEN] response, the client may use SEARCH UNSEEN to obtain that value. Leiba Informational [Page 6] RFC 2683 IMAP4 Implementation Recommendations September 1999 This mechanism is good as a default presentation method, but only works well if the default message order is acceptable. A client may want to present various sort orders to the user (by subject, by date sent, by sender, and so on) and in that case (lacking a SORT extension on the server side) the client WILL have to retrieve all message descriptors. A client that provides this service should not do it by default and should inform the user of the costs of choosing this option for large mailboxes. 3.2.1.3. Fetching a Large Body Part The issue here is similar to the one for a list of messages. In the BODYSTRUCTURE response the client knows the size, in bytes, of the body part it plans to fetch. Suppose this is a 70 MB video clip. The client can use partial fetches to retrieve the body part in pieces, avoiding the problem of an uninterruptible 70 MB literal coming back from the server: C: 022 FETCH 3 BODY[1]<0.20000> S: * 3 FETCH (FLAGS(\Seen) BODY[1]<0> {20000} S: ...data...) S: 022 OK done C: 023 FETCH 3 BODY[1]<20001.20000> S: * 3 FETCH (BODY[1]<20001> {20000} S: ...data...) S: 023 OK done C: 024 FETCH 3 BODY[1]<40001.20000> ...etc... 3.2.1.4. BODYSTRUCTURE vs. Entire Messages Because FETCH BODYSTRUCTURE is necessary in order to determine the number of body parts, and, thus, whether a message has "attachments", clients often use FETCH FULL as their normal method of populating the user's view of a mailbox. The benefit is that the client can display a paperclip icon or some such indication along with the normal message summary. However, this comes at a significant cost with some server configurations. The parsing needed to generate the FETCH BODYSTRUCTURE response may be time-consuming compared with that needed for FETCH ENVELOPE. The client developer should consider this issue when deciding whether the ability to add a paperclip icon is worth the tradeoff in performance, especially with large mailboxes. Some clients, rather than using FETCH BODYSTRUCTURE, use FETCH BODY[] (or the equivalent FETCH RFC822) to retrieve the entire message. They then do the MIME parsing in the client. This may give the client slightly more flexibility in some areas (access, for instance, to header fields that aren't returned in the BODYSTRUCTURE and Leiba Informational [Page 7] RFC 2683 IMAP4 Implementation Recommendations September 1999 ENVELOPE responses), but it can cause severe performance problems by forcing the transfer of all body parts when the user might only want to see some of them - a user logged on by modem and reading a small text message with a large ZIP file attached may prefer to read the text only and save the ZIP file for later. Therefore, a client should not normally retrieve entire messages and should retrieve message body parts selectively. 3.2.1.5. Long Command Lines A client can wind up building a very long command line in an effort to try to be efficient about requesting information from a server. This can typically happen when a client builds a message set from selected messages and doesn't recognise that contiguous blocks of messages may be group in a range. Suppose a user selects all 10,000 messages in a large mailbox and then unselects message 287. The client could build that message set as "1:286,288:10000", but a client that doesn't handle that might try to enumerate each message individually and build "1,2,3,4, [and so on] ,9999,10000". Adding that to the fetch command results in a command line that's almost 49,000 octets long, and, clearly, one can construct a command line that's even longer. A client should limit the length of the command lines it generates to approximately 1000 octets (including all quoted strings but not including literals). If the client is unable to group things into ranges so that the command line is within that length, it should split the request into multiple commands. The client should use literals instead of long quoted strings, in order to keep the command length down. For its part, a server should allow for a command line of at least 8000 octets. This provides plenty of leeway for accepting reasonable length commands from clients. The server should send a BAD response to a command that does not end within the server's maximum accepted command length. 3.2.2. Subscriptions The client isn't the only entity that can get flooded: the end user, too, may need some flood control. The IMAP4 protocol provides such control in the form of subscriptions. Most servers support the SUBSCRIBE, UNSUBSCRIBE, and LSUB commands, and many users choose to narrow down a large list of available mailboxes by subscribing to the ones that they usually want to see. Clients, with this in mind, should give the user a way to see only subscribed mailboxes. A client that never uses the LSUB command takes a significant usability feature away from the user. Of course, the client would not want to hide the LIST command completely; the user needs to have a way to Leiba Informational [Page 8] RFC 2683 IMAP4 Implementation Recommendations September 1999 choose between LIST and LSUB. The usual way to do this is to provide a setting like "show which mailboxes?: [] all [] subscribed only". 3.2.3. Searching IMAP SEARCH commands can become particularly troublesome (that is, slow) on mailboxes containing a large number of messages. So let's put a few things in perspective in that regard. The flag searches should be fast. The flag searches (ALL, [UN]SEEN, [UN]ANSWERED, [UN]DELETED, [UN]DRAFT, [UN]FLAGGED, NEW, OLD, RECENT) are known to be used by clients for the client's own use (for instance, some clients use "SEARCH UNSEEN" to find unseen mail and "SEARCH DELETED" to warn the user before expunging messages). Other searches, particularly the text searches (HEADER, TEXT, BODY) are initiated by the user, rather than by the client itself, and somewhat slower performance can be tolerated, since the user is aware that the search is being done (and is probably aware that it might be time-consuming). A smart server might use dynamic indexing to speed commonly used text searches. The client may allow other commands to be sent to the server while a SEARCH is in progress, but at the time of this writing there is little or no server support for parallel processing of multiple commands in the same session (and see "Multiple Accesses of the Same Mailbox" above for a description of the dangers of trying to work around this by doing your SEARCH in another session). Another word about text searches: some servers, built on database back-ends with indexed search capabilities, may return search results that do not match the IMAP spec's "case-insensitive substring" requirements. While these servers are in violation of the protocol, there is little harm in the violation as long as the search results are used only in response to a user's request. Still, developers of such servers should be aware that they ARE violating the protocol, should think carefully about that behaviour, and must be certain that their servers respond accurately to the flag searches for the reasons outlined above. In addition, servers should support CHARSET UTF-8 [UTF-8] in searches. Leiba Informational [Page 9] RFC 2683 IMAP4 Implementation Recommendations September 1999 3.3 Avoiding Invalid Requests IMAP4 provides ways for a server to tell a client in advance what is and isn't permitted in some circumstances. Clients should use these features to avoid sending requests that a well designed client would know to be invalid. This section explains this in more detail. 3.3.1. The CAPABILITY Command All IMAP4 clients should use the CAPABILITY command to determine what version of IMAP and what optional features a server supports. The client should not send IMAP4rev1 commands and arguments to a server that does not advertize IMAP4rev1 in its CAPABILITY response. Similarly, the client should not send IMAP4 commands that no longer exist in IMAP4rev1 to a server that does not advertize IMAP4 in its CAPABILITY response. An IMAP4rev1 server is NOT required to support obsolete IMAP4 or IMAP2bis commands (though some do; do not let this fact lull you into thinking that it's valid to send such commands to an IMAP4rev1 server). A client should not send commands to probe for the existance of certain extensions. All standard and standards-track extensions include CAPABILITY tokens indicating their presense. All private and experimental extensions should do the same, and clients that take advantage of them should use the CAPABILITY response to determine whether they may be used or not. 3.3.2. Don't Do What the Server Says You Can't In many cases, the server, in response to a command, will tell the client something about what can and can't be done with a particular mailbox. The client should pay attention to this information and should not try to do things that it's been told it can't do. Examples: * Do not try to SELECT a mailbox that has the \Noselect flag set. * Do not try to CREATE a sub-mailbox in a mailbox that has the \Noinferiors flag set. * Do not respond to a failing COPY or APPEND command by trying to CREATE the target mailbox if the server does not respond with a [TRYCREATE] response code. * Do not try to expunge a mailbox that has been selected with the [READ-ONLY] response code. Leiba Informational [Page 10] RFC 2683 IMAP4 Implementation Recommendations September 1999 3.4. Miscellaneous Protocol Considerations We describe here a number of important protocol-related issues, the misunderstanding of which has caused significant interoperability problems in IMAP4 implementations. One general item is that every implementer should be certain to take note of and to understand section 2.2.2 and the preamble to section 7 of the IMAP4rev1 spec [RFC-2060]. 3.4.1. Well Formed Protocol We cannot stress enough the importance of adhering strictly to the protocol grammar. The specification of the protocol is quite rigid; do not assume that you can insert blank space for "readability" if none is called for. Keep in mind that there are parsers out there that will crash if there are protocol errors. There are clients that will report every parser burp to the user. And in any case, information that cannot be parsed is information that is lost. Be careful in your protocol generation. And see "A Word About Testing", below. In particular, note that the string in the INTERNALDATE response is NOT an RFC-822 date string - that is, it is not in the same format as the first string in the ENVELOPE response. Since most clients will, in fact, accept an RFC-822 date string in the INTERNALDATE response, it's easy to miss this in your interoperability testing. But it will cause a problem with some client, so be sure to generate the correct string for this field. 3.4.2. Special Characters Certain characters, currently the double-quote and the backslash, may not be sent as-is inside a quoted string. These characters must be preceded by the escape character if they are in a quoted string, or else the string must be sent as a literal. Both clients and servers must handle this, both on output (they must send these characters properly) and on input (they must be able to receive escaped characters in quoted strings). Example: C: 001 LIST "" % S: * LIST () "" INBOX S: * LIST () "\\" TEST S: * LIST () "\\" {12} S: "My" mailbox S: 001 OK done C: 002 LIST "" "\"My\" mailbox\\%" S: * LIST () "\\" {17} S: "My" mailbox\Junk Leiba Informational [Page 11] RFC 2683 IMAP4 Implementation Recommendations September 1999 S: 002 OK done Note that in the example the server sent the hierarchy delimiter as an escaped character in the quoted string and sent the mailbox name containing imbedded double-quotes as a literal. The client used only quoted strings, escaping both the backslash and the double-quote characters. The CR and LF characters may be sent ONLY in literals; they are not allowed, even if escaped, inside quoted strings. And while we're talking about special characters: the IMAP spec, in the section titled "Mailbox International Naming Convention", describes how to encode mailbox names in modified UTF-7 [UTF-7 and RFC-2060]. Implementations must adhere to this in order to be interoperable in the international market, and servers should validate mailbox names sent by client and reject names that do not conform. As to special characters in userids and passwords: clients must not restrict what a user may type in for a userid or a password. The formal grammar specifies that these are "astrings", and an astring can be a literal. A literal, in turn can contain any 8-bit character, and clients must allow users to enter all 8-bit characters here, and must pass them, unchanged, to the server (being careful to send them as literals when necessary). In particular, some server configurations use "@" in user names, and some clients do not allow that character to be entered; this creates a severe interoperability problem. 3.4.3. UIDs and UIDVALIDITY Servers that support existing back-end mail stores often have no good place to save UIDs for messages. Often the existing mail store will not have the concept of UIDs in the sense that IMAP has: strictly increasing, never re-issued, 32-bit integers. Some servers solve this by storing the UIDs in a place that's accessible to end users, allowing for the possibility that the users will delete them. Others solve it by re-assigning UIDs every time a mailbox is selected. The server should maintain UIDs permanently for all messages if it can. If that's not possible, the server must change the UIDVALIDITY value for the mailbox whenever any of the UIDs may have become invalid. Clients must recognize that the UIDVALIDITY has changed and must respond to that condition by throwing away any information that they have saved about UIDs in that mailbox. There have been many problems in this area when clients have failed to do this; in the worst case it will result in loss of mail when a client deletes the Leiba Informational [Page 12] RFC 2683 IMAP4 Implementation Recommendations September 1999 wrong piece of mail by using a stale UID. It seems to be a common misunderstanding that "the UIDVALIDITY and the UID, taken together, form a 64-bit identifier that uniquely identifies a message on a server". This is absolutely NOT TRUE. There is no assurance that the UIDVALIDITY values of two mailboxes be different, so the UIDVALIDITY in no way identifies a mailbox. The ONLY purpose of UIDVALIDITY is, as its name indicates, to give the client a way to check the validity of the UIDs it has cached. While it is a valid implementation choice to put these values together to make a 64-bit identifier for the message, the important concept here is that UIDs are not unique between mailboxes; they are only unique WITHIN a given mailbox. Some server implementations have attempted to make UIDs unique across the entire server. This is inadvisable, in that it limits the life of UIDs unnecessarily. The UID is a 32-bit number and will run out in reasonably finite time if it's global across the server. If you assign UIDs sequentially in one mailbox, you will not have to start re-using them until you have had, at one time or another, 2**32 different messages in that mailbox. In the global case, you will have to reuse them once you have had, at one time or another, 2**32 different messages in the entire mail store. Suppose your server has around 8000 users registered (2**13). That gives an average of 2**19 UIDs per user. Suppose each user gets 32 messages (2**5) per day. That gives you 2**14 days (16000+ days = about 45 years) before you run out. That may seem like enough, but multiply the usage just a little (a lot of spam, a lot of mailing list subscriptions, more users) and you limit yourself too much. What's worse is that if you have to wrap the UIDs, and, thus, you have to change UIDVALIDITY and invalidate the UIDs in the mailbox, you have to do it for EVERY mailbox in the system, since they all share the same UID pool. If you assign UIDs per mailbox and you have a problem, you only have to kill the UIDs for that one mailbox. Under extreme circumstances (and this is extreme, indeed), the server may have to invalidate UIDs while a mailbox is in use by a client - that is, the UIDs that the client knows about in its active mailbox are no longer valid. In that case, the server must immediately change the UIDVALIDITY and must communicate this to the client. The server may do this by sending an unsolicited UIDVALIDITY message, in the same form as in response to the SELECT command. Clients must be prepared to handle such a message and the possibly coincident failure of the command in process. For example: Leiba Informational [Page 13] RFC 2683 IMAP4 Implementation Recommendations September 1999 C: 032 UID STORE 382 +Flags.silent \Deleted S: * OK [UIDVALIDITY 12345] New UIDVALIDITY value! S: 032 NO UID command rejected because UIDVALIDITY changed! C: ...invalidates local information and re-fetches... C: 033 FETCH 1:* UID ...etc... At the time of the writing of this document, the only server known to do this does so only under the following condition: the client selects INBOX, but there is not yet a physical INBOX file created. Nonetheless, the SELECT succeeds, exporting an empty INBOX with a temporary UIDVALIDITY of 1. While the INBOX remains selected, mail is delivered to the user, which creates the real INBOX file and assigns a permanent UIDVALIDITY (that is likely not to be 1). The server reports the change of UIDVALIDITY, but as there were no messages before, so no UIDs have actually changed, all the client must do is accept the change in UIDVALIDITY. Alternatively, a server may force the client to re-select the mailbox, at which time it will obtain a new UIDVALIDITY value. To do this, the server closes this client session (see "Severed Connections" above) and the client then reconnects and gets back in synch. Clients must be prepared for either of these behaviours. We do not know of, nor do we anticipate the future existance of, a server that changes UIDVALIDITY while there are existing messages, but clients must be prepared to handle this eventuality. 3.4.4. FETCH Responses When a client asks for certain information in a FETCH command, the server may return the requested information in any order, not necessarily in the order that it was requested. Further, the server may return the information in separate FETCH responses and may also return information that was not explicitly requested (to reflect to the client changes in the state of the subject message). Some examples: C: 001 FETCH 1 UID FLAGS INTERNALDATE S: * 5 FETCH (FLAGS (\Deleted)) S: * 1 FETCH (FLAGS (\Seen) INTERNALDATE "..." UID 345) S: 001 OK done (In this case, the responses are in a different order. Also, the server returned a flag update for message 5, which wasn't part of the client's request.) Leiba Informational [Page 14] RFC 2683 IMAP4 Implementation Recommendations September 1999 C: 002 FETCH 2 UID FLAGS INTERNALDATE S: * 2 FETCH (INTERNALDATE "...") S: * 2 FETCH (UID 399) S: * 2 FETCH (FLAGS ()) S: 002 OK done (In this case, the responses are in a different order and were returned in separate responses.) C: 003 FETCH 2 BODY[1] S: * 2 FETCH (FLAGS (\Seen) BODY[1] {14} S: Hello world! S: ) S: 003 OK done (In this case, the FLAGS response was added by the server, since fetching the body part caused the server to set the \Seen flag.) Because of this characteristic a client must be ready to receive any FETCH response at any time and should use that information to update its local information about the message to which the FETCH response refers. A client must not assume that any FETCH responses will come in any particular order, or even that any will come at all. If after receiving the tagged response for a FETCH command the client finds that it did not get all of the information requested, the client should send a NOOP command to the server to ensure that the server has an opportunity to send any pending EXPUNGE responses to the client (see [RFC-2180]). 3.4.5. RFC822.SIZE Some back-end mail stores keep the mail in a canonical form, rather than retaining the original MIME format of the messages. This means that the server must reassemble the message to produce a MIME stream when a client does a fetch such as RFC822 or BODY[], requesting the entire message. It also may mean that the server has no convenient way to know the RFC822.SIZE of the message. Often, such a server will actually have to build the MIME stream to compute the size, only to throw the stream away and report the size to the client. When this is the case, some servers have chosen to estimate the size, rather than to compute it precisely. Such an estimate allows the client to display an approximate size to the user and to use the estimate in flood control considerations (q.v.), but requires that the client not use the size for things such as allocation of buffers, because those buffers might then be too small to hold the actual MIME stream. Instead, a client should use the size that's returned in the literal when you fetch the data. Leiba Informational [Page 15] RFC 2683 IMAP4 Implementation Recommendations September 1999 The protocol requires that the RFC822.SIZE value returned by the server be EXACT. Estimating the size is a protocol violation, and server designers must be aware that, despite the performance savings they might realize in using an estimate, this practice will cause some clients to fail in various ways. If possible, the server should compute the RFC822.SIZE for a particular message once, and then save it for later retrieval. If that's not possible, the server must compute the value exactly every time. Incorrect estimates do cause severe interoperability problems with some clients. 3.4.6. Expunged Messages If the server allows multiple connections to the same mailbox, it is often possible for messages to be expunged in one client unbeknownst to another client. Since the server is not allowed to tell the client about these expunged messages in response to a FETCH command, the server may have to deal with the issue of how to return information about an expunged message. There was extensive discussion about this issue, and the results of that discussion are summarized in [RFC-2180]. See that reference for a detailed explanation and for recommendations. 3.4.7. The Namespace Issue Namespaces are a very muddy area in IMAP4 implementation right now (see [NAMESPACE] for a proposal to clear the water a bit). Until the issue is resolved, the important thing for client developers to understand is that some servers provide access through IMAP to more than just the user's personal mailboxes, and, in fact, the user's personal mailboxes may be "hidden" somewhere in the user's default hierarchy. The client, therefore, should provide a setting wherein the user can specify a prefix to be used when accessing mailboxes. If the user's mailboxes are all in "~/mail/", for instance, then the user can put that string in the prefix. The client would then put the prefix in front of any name pattern in the LIST and LSUB commands: C: 001 LIST "" ~/mail/% (See also "Reference Names in the LIST Command" below.) 3.4.8. Creating Special-Use Mailboxes It may seem at first that this is part of the namespace issue; it is not, and is only indirectly related to it. A number of clients like to create special-use mailboxes with particular names. Most commonly, clients with a "trash folder" model of message deletion want to create a mailbox with the name "Trash" or "Deleted". Some Leiba Informational [Page 16] RFC 2683 IMAP4 Implementation Recommendations September 1999 clients want to create a "Drafts" mailbox, an "Outbox" mailbox, or a "Sent Mail" mailbox. And so on. There are two major interoperability problems with this practice: 1. different clients may use different names for mailboxes with similar functions (such as "Trash" and "Deleted"), or may manage the same mailboxes in different ways, causing problems if a user switches between clients and 2. there is no guarantee that the server will allow the creation of the desired mailbox. The client developer is, therefore, well advised to consider carefully the creation of any special-use mailboxes on the server, and, further, the client must not require such mailbox creation - that is, if you do decide to do this, you must handle gracefully the failure of the CREATE command and behave reasonably when your special-use mailboxes do not exist and can not be created. In addition, the client developer should provide a convenient way for the user to select the names for any special-use mailboxes, allowing the user to make these names the same in all clients used and to put them where the user wants them. 3.4.9. Reference Names in the LIST Command Many implementers of both clients and servers are confused by the "reference name" on the LIST command. The reference name is intended to be used in much the way a "cd" (change directory) command is used on Unix, PC DOS, Windows, and OS/2 systems. That is, the mailbox name is interpreted in much the same way as a file of that name would be found if one had done a "cd" command into the directory specified by the reference name. For example, in Unix we have the following: > cd /u/jones/junk > vi banana [file is "/u/jones/junk/banana"] > vi stuff/banana [file is "/u/jones/junk/stuff/banana"] > vi /etc/hosts [file is "/etc/hosts"] In the past, there have been several interoperability problems with this. First, while some IMAP servers are built on Unix or PC file systems, many others are not, and the file system semantics do not make sense in those configurations. Second, while some IMAP servers expose the underlying file system to the clients, others allow access only to the user's personal mailboxes, or to some other limited set of files, making such file-system-like semantics less meaningful. Third, because the IMAP spec leaves the interpretation of the reference name as "implementation-dependent", in the past the various server implementations handled it in vastly differing ways. Leiba Informational [Page 17] RFC 2683 IMAP4 Implementation Recommendations September 1999 The following recommendations are the result of significant operational experience, and are intended to maximize interoperability. Server implementations must implement the reference argument in a way that matches the intended "change directory" operation as closely as possible. As a minimum implementation, the reference argument may be prepended to the mailbox name (while suppressing double delimiters; see the next paragraph). Even servers that do not provide a way to break out of the current hierarchy (see "breakout facility" below) must provide a reasonable implementation of the reference argument, as described here, so that they will interoperate with clients that use it. Server implementations that prepend the reference argument to the mailbox name should insert a hierarchy delimiter between them, and must not insert a second if one is already present: C: A001 LIST ABC DEF S: * LIST () "/" ABC/DEF <=== should do this S: A001 OK done C: A002 LIST ABC/ /DEF S: * LIST () "/" ABC//DEF <=== must not do this S: A002 OK done On clients, the reference argument is chiefly used to implement a "breakout facility", wherein the user may directly access a mailbox outside the "current directory" hierarchy. Client implementations should have an operational mode that does not use the reference argument. This is to interoperate with older servers that did not implement the reference argument properly. While it's a good idea to give the user access to a breakout facility, clients that do not intend to do so should not use the reference argument at all. Client implementations should always place a trailing hierarchy delimiter on the reference argument. This is because some servers prepend the reference argument to the mailbox name without inserting a hierarchy delimiter, while others do insert a hierarchy delimiter if one is not already present. A client that puts the delimiter in will work with both varieties of server. Client implementations that implement a breakout facility should allow the user to choose whether or not to use a leading hierarchy delimiter on the mailbox argument. This is because the handling of a leading mailbox hierarchy delimiter also varies from server to server, and even between different mailstores on the same server. In some cases, a leading hierarchy delimiter means "discard the Leiba Informational [Page 18] RFC 2683 IMAP4 Implementation Recommendations September 1999 reference argument" (implementing the intended breakout facility), thus: C: A001 LIST ABC/ /DEF S: * LIST () "/" /DEF S: A001 OK done In other cases, however, the two are catenated and the extra hierarchy delimiter is discarded, thus: C: A001 LIST ABC/ /DEF S: * LIST () "/" ABC/DEF S: A001 OK done Client implementations must not assume that the server supports a breakout facility, but may provide a way for the user to use one if it is available. Any breakout facility should be exported to the user interface. Note that there may be other "breakout" characters besides the hierarchy delimiter (for instance, UNIX filesystem servers are likely to use a leading "~" as well), and that their interpretation is server-dependent. 3.4.10. Mailbox Hierarchy Delimiters The server's selection of what to use as a mailbox hierarchy delimiter is a difficult one, involving several issues: What characters do users expect to see? What characters can they enter for a hierarchy delimiter if it is desired (or required) that the user enter it? What character can be used for the hierarchy delimiter, noting that the chosen character can not otherwise be used in the mailbox name? Because some interfaces show users the hierarchy delimiters or allow users to enter qualified mailbox names containing them, server implementations should use delimiter characters that users generally expect to see as name separators. The most common characters used for this are "/" (as in Unix file names), "\" (as in OS/2 and Windows file names), and "." (as in news groups). There is little to choose among these apart from what users may expect or what is dictated by the underlying file system, if any. One consideration about using "\" is that it's also a special character in the IMAP protocol. While the use of other hierarchy delimiter characters is permissible, A DESIGNER IS WELL ADVISED TO STAY WITH ONE FROM THIS SET unless the server is intended for special purposes only. Implementers might be thinking about using characters such as "-", "_", ";", "&", "#", "@", and "!", but they should be aware of the surprise to the user as well as of the effect on URLs and other external specifications (since some of these characters have special meanings there). Also, a Leiba Informational [Page 19] RFC 2683 IMAP4 Implementation Recommendations September 1999 server that uses "\" (and clients of such a server) must remember to escape that character in quoted strings or to send literals instead. Literals are recommended over escaped characters in quoted strings in order to maintain compatibility with older IMAP versions that did not allow escaped characters in quoted strings (but check the grammar to see where literals are allowed): C: 001 LIST "" {13} S: + send literal C: this\%\%\%\h* S: * LIST () "\\" {27} S: this\is\a\mailbox\hierarchy S: 001 OK LIST complete In any case, a server should not use normal alpha-numeric characters (such as "X" or "0") as delimiters; a user would be very surprised to find that "EXPENDITURES" actually represented a two-level hierarchy. And a server should not use characters that are non-printable or difficult or impossible to enter on a standard US keyboard. Control characters, box-drawing characters, and characters from non-US alphabets fit into this category. Their use presents interoperability problems that are best avoided. The UTF-7 encoding of mailbox names also raises questions about what to do with the hierarchy delimiters in encoded names: do we encode each hierarchy level and separate them with delimiters, or do we encode the fully qualified name, delimiters and all? The answer for IMAP is the former: encode each hierarchy level separately, and insert delimiters between. This makes it particularly important not to use as a hierarchy delimiter a character that might cause confusion with IMAP's modified UTF-7 [UTF-7 and RFC-2060] encoding. To repeat: a server should use "/", "\", or "." as its hierarchy delimiter. The use of any other character is likely to cause problems and is STRONGLY DISCOURAGED. 3.4.11. ALERT Response Codes The protocol spec is very clear on the matter of what to do with ALERT response codes, and yet there are many clients that violate it so it needs to be said anyway: "The human-readable text contains a special alert that must be presented to the user in a fashion that calls the user's attention to the message." That should be clear enough, but I'll repeat it here: Clients must present ALERT text clearly to the user. Leiba Informational [Page 20] RFC 2683 IMAP4 Implementation Recommendations September 1999 3.4.12. Deleting Mailboxes The protocol does not guarantee that a client may delete a mailbox that is not empty, though on some servers it is permissible and is, in fact, much faster than the alternative or deleting all the messages from the client. If the client chooses to try to take advantage of this possibility it must be prepared to use the other method in the even that the more convenient one fails. Further, a client should not try to delete the mailbox that it has selected, but should first close that mailbox; some servers do not permit the deletion of the selected mailbox. That said, a server should permit the deletion of a non-empty mailbox; there's little reason to pass this work on to the client. Moreover, forbidding this prevents the deletion of a mailbox that for some reason can not be opened or expunged, leading to possible denial-of-service problems. Example: [User tells the client to delete mailbox BANANA, which is currently selected...] C: 008 CLOSE S: 008 OK done C: 009 DELETE BANANA S: 009 NO Delete failed; mailbox is not empty. C: 010 SELECT BANANA S: * ... untagged SELECT responses S: 010 OK done C: 011 STORE 1:* +FLAGS.SILENT \DELETED S: 011 OK done C: 012 CLOSE S: 012 OK done C: 013 DELETE BANANA S: 013 OK done 3.5. A Word About Testing Since the whole point of IMAP is interoperability, and since interoperability can not be tested in a vacuum, the final recommendation of this treatise is, "Test against EVERYTHING." Test your client against every server you can get an account on. Test your server with every client you can get your hands on. Many clients make limited test versions available on the Web for the downloading. Many server owners will give serious client developers guest accounts for testing. Contact them and ask. NEVER assume that because your client works with one or two servers, or because your server does fine with one or two clients, you will interoperate well Leiba Informational [Page 21] RFC 2683 IMAP4 Implementation Recommendations September 1999 in general. In particular, in addition to everything else, be sure to test against the reference implementations: the PINE client, the University of Washington server, and the Cyrus server. See the following URLs on the web for more information here: IMAP Products and Sources: http://www.imap.org/products.html IMC MailConnect: http://www.imc.org/imc-mailconnect 4. Security Considerations This document describes behaviour of clients and servers that use the IMAP4 protocol, and as such, has the same security considerations as described in [RFC-2060]. 5. References [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC 2180, July 1997. [UTF-8] Yergeau, F., " UTF-8, a transformation format of Unicode and ISO 10646", RFC 2044, October 1996. [UTF-7] Goldsmith, D. and M. Davis, "UTF-7, a Mail-Safe Transformation Format of Unicode", RFC 2152, May 1997. [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", Work in Progress. 6. Author's Address Barry Leiba IBM T.J. Watson Research Center 30 Saw Mill River Road Hawthorne, NY 10532 Phone: 1-914-784-7941 EMail: leiba@watson.ibm.com Leiba Informational [Page 22] RFC 2683 IMAP4 Implementation Recommendations September 1999 7. Full Copyright Statement Copyright (C) The Internet Society (1999). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Leiba Informational [Page 23] ================================================ FILE: docs/rfcs/rfc2831.Obsolete_Digest_AUTHentication_as_a_SASL_mech.txt ================================================ Network Working Group P. Leach Request for Comments: 2831 Microsoft Category: Standards Track C. Newman Innosoft May 2000 Using Digest Authentication as a SASL Mechanism Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2000). All Rights Reserved. Abstract This specification defines how HTTP Digest Authentication [Digest] can be used as a SASL [RFC 2222] mechanism for any protocol that has a SASL profile. It is intended both as an improvement over CRAM-MD5 [RFC 2195] and as a convenient way to support a single authentication mechanism for web, mail, LDAP, and other protocols. Table of Contents 1 INTRODUCTION.....................................................2 1.1 CONVENTIONS AND NOTATION......................................2 1.2 REQUIREMENTS..................................................3 2 AUTHENTICATION...................................................3 2.1 INITIAL AUTHENTICATION........................................3 2.1.1 Step One...................................................3 2.1.2 Step Two...................................................6 2.1.3 Step Three................................................12 2.2 SUBSEQUENT AUTHENTICATION....................................12 2.2.1 Step one..................................................13 2.2.2 Step Two..................................................13 2.3 INTEGRITY PROTECTION.........................................13 2.4 CONFIDENTIALITY PROTECTION...................................14 3 SECURITY CONSIDERATIONS.........................................15 3.1 AUTHENTICATION OF CLIENTS USING DIGEST AUTHENTICATION........15 3.2 COMPARISON OF DIGEST WITH PLAINTEXT PASSWORDS................16 3.3 REPLAY ATTACKS...............................................16 Leach & Newman Standards Track [Page 1] RFC 2831 Digest SASL Mechanism May 2000 3.4 ONLINE DICTIONARY ATTACKS....................................16 3.5 OFFLINE DICTIONARY ATTACKS...................................16 3.6 MAN IN THE MIDDLE............................................17 3.7 CHOSEN PLAINTEXT ATTACKS.....................................17 3.8 SPOOFING BY COUNTERFEIT SERVERS..............................17 3.9 STORING PASSWORDS............................................17 3.10 MULTIPLE REALMS.............................................18 3.11 SUMMARY.....................................................18 4 EXAMPLE.........................................................18 5 REFERENCES......................................................20 6 AUTHORS' ADDRESSES..............................................21 7 ABNF............................................................21 7.1 AUGMENTED BNF................................................21 7.2 BASIC RULES..................................................23 8 SAMPLE CODE.....................................................25 9 FULL COPYRIGHT STATEMENT........................................27 1 Introduction This specification describes the use of HTTP Digest Access Authentication as a SASL mechanism. The authentication type associated with the Digest SASL mechanism is "DIGEST-MD5". This specification is intended to be upward compatible with the "md5-sess" algorithm of HTTP/1.1 Digest Access Authentication specified in [Digest]. The only difference in the "md5-sess" algorithm is that some directives not needed in a SASL mechanism have had their values defaulted. There is one new feature for use as a SASL mechanism: integrity protection on application protocol messages after an authentication exchange. Also, compared to CRAM-MD5, DIGEST-MD5 prevents chosen plaintext attacks, and permits the use of third party authentication servers, mutual authentication, and optimized reauthentication if a client has recently authenticated to a server. 1.1 Conventions and Notation This specification uses the same ABNF notation and lexical conventions as HTTP/1.1 specification; see appendix A. Let { a, b, ... } be the concatenation of the octet strings a, b, ... Let H(s) be the 16 octet MD5 hash [RFC 1321] of the octet string s. Leach & Newman Standards Track [Page 2] RFC 2831 Digest SASL Mechanism May 2000 Let KD(k, s) be H({k, ":", s}), i.e., the 16 octet hash of the string k, a colon and the string s. Let HEX(n) be the representation of the 16 octet MD5 hash n as a string of 32 hex digits (with alphabetic characters always in lower case, since MD5 is case sensitive). Let HMAC(k, s) be the 16 octet HMAC-MD5 [RFC 2104] of the octet string s using the octet string k as a key. The value of a quoted string constant as an octet string does not include any terminating null character. 1.2 Requirements The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [RFC 2119]. An implementation is not compliant if it fails to satisfy one or more of the MUST level requirements for the protocols it implements. An implementation that satisfies all the MUST level and all the SHOULD level requirements for its protocols is said to be "unconditionally compliant"; one that satisfies all the MUST level requirements but not all the SHOULD level requirements for its protocols is said to be "conditionally compliant." 2 Authentication The following sections describe how to use Digest as a SASL authentication mechanism. 2.1 Initial Authentication If the client has not recently authenticated to the server, then it must perform "initial authentication", as defined in this section. If it has recently authenticated, then a more efficient form is available, defined in the next section. 2.1.1 Step One The server starts by sending a challenge. The data encoded in the challenge contains a string formatted according to the rules for a "digest-challenge" defined as follows: Leach & Newman Standards Track [Page 3] RFC 2831 Digest SASL Mechanism May 2000 digest-challenge = 1#( realm | nonce | qop-options | stale | maxbuf | charset algorithm | cipher-opts | auth-param ) realm = "realm" "=" <"> realm-value <"> realm-value = qdstr-val nonce = "nonce" "=" <"> nonce-value <"> nonce-value = qdstr-val qop-options = "qop" "=" <"> qop-list <"> qop-list = 1#qop-value qop-value = "auth" | "auth-int" | "auth-conf" | token stale = "stale" "=" "true" maxbuf = "maxbuf" "=" maxbuf-value maxbuf-value = 1*DIGIT charset = "charset" "=" "utf-8" algorithm = "algorithm" "=" "md5-sess" cipher-opts = "cipher" "=" <"> 1#cipher-value <"> cipher-value = "3des" | "des" | "rc4-40" | "rc4" | "rc4-56" | token auth-param = token "=" ( token | quoted-string ) The meanings of the values of the directives used above are as follows: realm Mechanistically, a string which can enable users to know which username and password to use, in case they might have different ones for different servers. Conceptually, it is the name of a collection of accounts that might include the user's account. This string should contain at least the name of the host performing the authentication and might additionally indicate the collection of users who might have access. An example might be "registered_users@gotham.news.example.com". This directive is optional; if not present, the client SHOULD solicit it from the user or be able to compute a default; a plausible default might be the realm supplied by the user when they logged in to the client system. Multiple realm directives are allowed, in which case the user or client must choose one as the realm for which to supply to username and password. nonce A server-specified data string which MUST be different each time a digest-challenge is sent as part of initial authentication. It is recommended that this string be base64 or hexadecimal data. Note that since the string is passed as a quoted string, the double-quote character is not allowed unless escaped (see section 7.2). The contents of the nonce are implementation dependent. The Leach & Newman Standards Track [Page 4] RFC 2831 Digest SASL Mechanism May 2000 security of the implementation depends on a good choice. It is RECOMMENDED that it contain at least 64 bits of entropy. The nonce is opaque to the client. This directive is required and MUST appear exactly once; if not present, or if multiple instances are present, the client should abort the authentication exchange. qop-options A quoted string of one or more tokens indicating the "quality of protection" values supported by the server. The value "auth" indicates authentication; the value "auth-int" indicates authentication with integrity protection; the value "auth-conf" indicates authentication with integrity protection and encryption. This directive is optional; if not present it defaults to "auth". The client MUST ignore unrecognized options; if the client recognizes no option, it should abort the authentication exchange. stale The "stale" directive is not used in initial authentication. See the next section for its use in subsequent authentications. This directive may appear at most once; if multiple instances are present, the client should abort the authentication exchange. maxbuf A number indicating the size of the largest buffer the server is able to receive when using "auth-int" or "auth-conf". If this directive is missing, the default value is 65536. This directive may appear at most once; if multiple instances are present, the client should abort the authentication exchange. charset This directive, if present, specifies that the server supports UTF-8 encoding for the username and password. If not present, the username and password must be encoded in ISO 8859-1 (of which US-ASCII is a subset). The directive is needed for backwards compatibility with HTTP Digest, which only supports ISO 8859-1. This directive may appear at most once; if multiple instances are present, the client should abort the authentication exchange. algorithm This directive is required for backwards compatibility with HTTP Digest., which supports other algorithms. . This directive is required and MUST appear exactly once; if not present, or if multiple instances are present, the client should abort the authentication exchange. Leach & Newman Standards Track [Page 5] RFC 2831 Digest SASL Mechanism May 2000 cipher-opts A list of ciphers that the server supports. This directive must be present exactly once if "auth-conf" is offered in the "qop-options" directive, in which case the "3des" and "des" modes are mandatory-to-implement. The client MUST ignore unrecognized options; if the client recognizes no option, it should abort the authentication exchange. des the Data Encryption Standard (DES) cipher [FIPS] in cipher block chaining (CBC) mode with a 56 bit key. 3des the "triple DES" cipher in CBC mode with EDE with the same key for each E stage (aka "two keys mode") for a total key length of 112 bits. rc4, rc4-40, rc4-56 the RC4 cipher with a 128 bit, 40 bit, and 56 bit key, respectively. auth-param This construct allows for future extensions; it may appear more than once. The client MUST ignore any unrecognized directives. For use as a SASL mechanism, note that the following changes are made to "digest-challenge" from HTTP: the following Digest options (called "directives" in HTTP terminology) are unused (i.e., MUST NOT be sent, and MUST be ignored if received): opaque domain The size of a digest-challenge MUST be less than 2048 bytes. 2.1.2 Step Two The client makes note of the "digest-challenge" and then responds with a string formatted and computed according to the rules for a "digest-response" defined as follows: Leach & Newman Standards Track [Page 6] RFC 2831 Digest SASL Mechanism May 2000 digest-response = 1#( username | realm | nonce | cnonce | nonce-count | qop | digest-uri | response | maxbuf | charset | cipher | authzid | auth-param ) username = "username" "=" <"> username-value <"> username-value = qdstr-val cnonce = "cnonce" "=" <"> cnonce-value <"> cnonce-value = qdstr-val nonce-count = "nc" "=" nc-value nc-value = 8LHEX qop = "qop" "=" qop-value digest-uri = "digest-uri" "=" <"> digest-uri-value <"> digest-uri-value = serv-type "/" host [ "/" serv-name ] serv-type = 1*ALPHA host = 1*( ALPHA | DIGIT | "-" | "." ) serv-name = host response = "response" "=" response-value response-value = 32LHEX LHEX = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "a" | "b" | "c" | "d" | "e" | "f" cipher = "cipher" "=" cipher-value authzid = "authzid" "=" <"> authzid-value <"> authzid-value = qdstr-val username The user's name in the specified realm, encoded according to the value of the "charset" directive. This directive is required and MUST be present exactly once; otherwise, authentication fails. realm The realm containing the user's account. This directive is required if the server provided any realms in the "digest-challenge", in which case it may appear exactly once and its value SHOULD be one of those realms. If the directive is missing, "realm-value" will set to the empty string when computing A1 (see below for details). nonce The server-specified data string received in the preceding digest-challenge. This directive is required and MUST be present exactly once; otherwise, authentication fails. Leach & Newman Standards Track [Page 7] RFC 2831 Digest SASL Mechanism May 2000 cnonce A client-specified data string which MUST be different each time a digest-response is sent as part of initial authentication. The cnonce-value is an opaque quoted string value provided by the client and used by both client and server to avoid chosen plaintext attacks, and to provide mutual authentication. The security of the implementation depends on a good choice. It is RECOMMENDED that it contain at least 64 bits of entropy. This directive is required and MUST be present exactly once; otherwise, authentication fails. nonce-count The nc-value is the hexadecimal count of the number of requests (including the current request) that the client has sent with the nonce value in this request. For example, in the first request sent in response to a given nonce value, the client sends "nc=00000001". The purpose of this directive is to allow the server to detect request replays by maintaining its own copy of this count - if the same nc-value is seen twice, then the request is a replay. See the description below of the construction of the response value. This directive may appear at most once; if multiple instances are present, the client should abort the authentication exchange. qop Indicates what "quality of protection" the client accepted. If present, it may appear exactly once and its value MUST be one of the alternatives in qop-options. If not present, it defaults to "auth". These values affect the computation of the response. Note that this is a single token, not a quoted list of alternatives. serv-type Indicates the type of service, such as "www" for web service, "ftp" for FTP service, "smtp" for mail delivery service, etc. The service name as defined in the SASL profile for the protocol see section 4 of [RFC 2222], registered in the IANA registry of "service" elements for the GSSAPI host-based service name form [RFC 2078]. host The DNS host name or IP address for the service requested. The DNS host name must be the fully-qualified canonical name of the host. The DNS host name is the preferred form; see notes on server processing of the digest-uri. Leach & Newman Standards Track [Page 8] RFC 2831 Digest SASL Mechanism May 2000 serv-name Indicates the name of the service if it is replicated. The service is considered to be replicated if the client's service-location process involves resolution using standard DNS lookup operations, and if these operations involve DNS records (such as SRV, or MX) which resolve one DNS name into a set of other DNS names. In this case, the initial name used by the client is the "serv-name", and the final name is the "host" component. For example, the incoming mail service for "example.com" may be replicated through the use of MX records stored in the DNS, one of which points at an SMTP server called "mail3.example.com"; it's "serv-name" would be "example.com", it's "host" would be "mail3.example.com". If the service is not replicated, or the serv-name is identical to the host, then the serv-name component MUST be omitted. digest-uri Indicates the principal name of the service with which the client wishes to connect, formed from the serv-type, host, and serv-name. For example, the FTP service on "ftp.example.com" would have a "digest-uri" value of "ftp/ftp.example.com"; the SMTP server from the example above would have a "digest-uri" value of "smtp/mail3.example.com/example.com". Servers SHOULD check that the supplied value is correct. This will detect accidental connection to the incorrect server. It is also so that clients will be trained to provide values that will work with implementations that use a shared back-end authentication service that can provide server authentication. The serv-type component should match the service being offered. The host component should match one of the host names of the host on which the service is running, or it's IP address. Servers SHOULD NOT normally support the IP address form, because server authentication by IP address is not very useful; they should only do so if the DNS is unavailable or unreliable. The serv-name component should match one of the service's configured service names. This directive may appear at most once; if multiple instances are present, the client should abort the authentication exchange. Note: In the HTTP use of Digest authentication, the digest-uri is the URI (usually a URL) of the resource requested -- hence the name of the directive. response A string of 32 hex digits computed as defined below, which proves that the user knows a password. This directive is required and MUST be present exactly once; otherwise, authentication fails. Leach & Newman Standards Track [Page 9] RFC 2831 Digest SASL Mechanism May 2000 maxbuf A number indicating the size of the largest buffer the client is able to receive. If this directive is missing, the default value is 65536. This directive may appear at most once; if multiple instances are present, the server should abort the authentication exchange. charset This directive, if present, specifies that the client has used UTF-8 encoding for the username and password. If not present, the username and password must be encoded in ISO 8859-1 (of which US-ASCII is a subset). The client should send this directive only if the server has indicated it supports UTF-8. The directive is needed for backwards compatibility with HTTP Digest, which only supports ISO 8859-1. LHEX 32 hex digits, where the alphabetic characters MUST be lower case, because MD5 is not case insensitive. cipher The cipher chosen by the client. This directive MUST appear exactly once if "auth-conf" is negotiated; if required and not present, authentication fails. authzid The "authorization ID" as per RFC 2222, encoded in UTF-8. This directive is optional. If present, and the authenticating user has sufficient privilege, and the server supports it, then after authentication the server will use this identity for making all accesses and access checks. If the client specifies it, and the server does not support it, then the response-value will be incorrect, and authentication will fail. The size of a digest-response MUST be less than 4096 bytes. 2.1.2.1 Response-value The definition of "response-value" above indicates the encoding for its value -- 32 lower case hex characters. The following definitions show how the value is computed. Although qop-value and components of digest-uri-value may be case-insensitive, the case which the client supplies in step two is preserved for the purpose of computing and verifying the response-value. response-value = Leach & Newman Standards Track [Page 10] RFC 2831 Digest SASL Mechanism May 2000 HEX( KD ( HEX(H(A1)), { nonce-value, ":" nc-value, ":", cnonce-value, ":", qop-value, ":", HEX(H(A2)) })) If authzid is specified, then A1 is A1 = { H( { username-value, ":", realm-value, ":", passwd } ), ":", nonce-value, ":", cnonce-value, ":", authzid-value } If authzid is not specified, then A1 is A1 = { H( { username-value, ":", realm-value, ":", passwd } ), ":", nonce-value, ":", cnonce-value } where passwd = *OCTET The "username-value", "realm-value" and "passwd" are encoded according to the value of the "charset" directive. If "charset=UTF-8" is present, and all the characters of either "username-value" or "passwd" are in the ISO 8859-1 character set, then it must be converted to ISO 8859-1 before being hashed. This is so that authentication databases that store the hashed username, realm and password (which is common) can be shared compatibly with HTTP, which specifies ISO 8859-1. A sample implementation of this conversion is in section 8. If the "qop" directive's value is "auth", then A2 is: A2 = { "AUTHENTICATE:", digest-uri-value } If the "qop" value is "auth-int" or "auth-conf" then A2 is: A2 = { "AUTHENTICATE:", digest-uri-value, ":00000000000000000000000000000000" } Note that "AUTHENTICATE:" must be in upper case, and the second string constant is a string with a colon followed by 32 zeros. These apparently strange values of A2 are for compatibility with HTTP; they were arrived at by setting "Method" to "AUTHENTICATE" and the hash of the entity body to zero in the HTTP digest calculation of A2. Also, in the HTTP usage of Digest, several directives in the Leach & Newman Standards Track [Page 11] RFC 2831 Digest SASL Mechanism May 2000 "digest-challenge" sent by the server have to be returned by the client in the "digest-response". These are: opaque algorithm These directives are not needed when Digest is used as a SASL mechanism (i.e., MUST NOT be sent, and MUST be ignored if received). 2.1.3 Step Three The server receives and validates the "digest-response". The server checks that the nonce-count is "00000001". If it supports subsequent authentication (see section 2.2), it saves the value of the nonce and the nonce-count. It sends a message formatted as follows: response-auth = "rspauth" "=" response-value where response-value is calculated as above, using the values sent in step two, except that if qop is "auth", then A2 is A2 = { ":", digest-uri-value } And if qop is "auth-int" or "auth-conf" then A2 is A2 = { ":", digest-uri-value, ":00000000000000000000000000000000" } Compared to its use in HTTP, the following Digest directives in the "digest-response" are unused: nextnonce qop cnonce nonce-count 2.2 Subsequent Authentication If the client has previously authenticated to the server, and remembers the values of username, realm, nonce, nonce-count, cnonce, and qop that it used in that authentication, and the SASL profile for a protocol permits an initial client response, then it MAY perform "subsequent authentication", as defined in this section. Leach & Newman Standards Track [Page 12] RFC 2831 Digest SASL Mechanism May 2000 2.2.1 Step one The client uses the values from the previous authentication and sends an initial response with a string formatted and computed according to the rules for a "digest-response", as defined above, but with a nonce-count one greater than used in the last "digest-response". 2.2.2 Step Two The server receives the "digest-response". If the server does not support subsequent authentication, then it sends a "digest-challenge", and authentication proceeds as in initial authentication. If the server has no saved nonce and nonce-count from a previous authentication, then it sends a "digest-challenge", and authentication proceeds as in initial authentication. Otherwise, the server validates the "digest-response", checks that the nonce-count is one greater than that used in the previous authentication using that nonce, and saves the new value of nonce-count. If the response is invalid, then the server sends a "digest-challenge", and authentication proceeds as in initial authentication (and should be configurable to log an authentication failure in some sort of security audit log, since the failure may be a symptom of an attack). The nonce-count MUST NOT be incremented in this case: to do so would allow a denial of service attack by sending an out-of-order nonce-count. If the response is valid, the server MAY choose to deem that authentication has succeeded. However, if it has been too long since the previous authentication, or for any other reason, the server MAY send a new "digest-challenge" with a new value for nonce. The challenge MAY contain a "stale" directive with value "true", which says that the client may respond to the challenge using the password it used in the previous response; otherwise, the client must solicit the password anew from the user. This permits the server to make sure that the user has presented their password recently. (The directive name refers to the previous nonce being stale, not to the last use of the password.) Except for the handling of "stale", after sending the "digest-challenge" authentication proceeds as in the case of initial authentication. 2.3 Integrity Protection If the server offered "qop=auth-int" and the client responded "qop=auth-int", then subsequent messages, up to but not including the next subsequent authentication, between the client and the server Leach & Newman Standards Track [Page 13] RFC 2831 Digest SASL Mechanism May 2000 MUST be integrity protected. Using as a base session key the value of H(A1) as defined above the client and server calculate a pair of message integrity keys as follows. The key for integrity protecting messages from client to server is: Kic = MD5({H(A1), "Digest session key to client-to-server signing key magic constant"}) The key for integrity protecting messages from server to client is: Kis = MD5({H(A1), "Digest session key to server-to-client signing key magic constant"}) where MD5 is as specified in [RFC 1321]. If message integrity is negotiated, a MAC block for each message is appended to the message. The MAC block is 16 bytes: the first 10 bytes of the HMAC-MD5 [RFC 2104] of the message, a 2-byte message type number in network byte order with value 1, and the 4-byte sequence number in network byte order. The message type is to allow for future extensions such as rekeying. MAC(Ki, SeqNum, msg) = (HMAC(Ki, {SeqNum, msg})[0..9], 0x0001, SeqNum) where Ki is Kic for messages sent by the client and Kis for those sent by the server. The sequence number is initialized to zero, and incremented by one for each message sent. Upon receipt, MAC(Ki, SeqNum, msg) is computed and compared with the received value; the message is discarded if they differ. 2.4 Confidentiality Protection If the server sent a "cipher-opts" directive and the client responded with a "cipher" directive, then subsequent messages between the client and the server MUST be confidentiality protected. Using as a base session key the value of H(A1) as defined above the client and server calculate a pair of message integrity keys as follows. The key for confidentiality protecting messages from client to server is: Kcc = MD5({H(A1)[0..n], "Digest H(A1) to client-to-server sealing key magic constant"}) The key for confidentiality protecting messages from server to client is: Leach & Newman Standards Track [Page 14] RFC 2831 Digest SASL Mechanism May 2000 Kcs = MD5({H(A1)[0..n], "Digest H(A1) to server-to-client sealing key magic constant"}) where MD5 is as specified in [RFC 1321]. For cipher "rc4-40" n is 5; for "rc4-56" n is 7; for the rest n is 16. The key for the "rc-*" ciphers is all 16 bytes of Kcc or Kcs; the key for "des" is the first 7 bytes; the key for "3des" is the first 14 bytes. The IV for "des" and "3des" is the last 8 bytes of Kcc or Kcs. If message confidentiality is negotiated, each message is encrypted with the chosen cipher and a MAC block is appended to the message. The MAC block is a variable length padding prefix followed by 16 bytes formatted as follows: the first 10 bytes of the HMAC-MD5 [RFC 2104] of the message, a 2-byte message type number in network byte order with value 1, and the 4-byte sequence number in network byte order. If the blocksize of the chosen cipher is not 1 byte, the padding prefix is one or more octets each containing the number of padding bytes, such that total length of the encrypted part of the message is a multiple of the blocksize. The padding and first 10 bytes of the MAC block are encrypted along with the message. SEAL(Ki, Kc, SeqNum, msg) = {CIPHER(Kc, {msg, pad, HMAC(Ki, {SeqNum, msg})[0..9])}), 0x0001, SeqNum} where CIPHER is the chosen cipher, Ki and Kc are Kic and Kcc for messages sent by the client and Kis and Kcs for those sent by the server. The sequence number is initialized to zero, and incremented by one for each message sent. Upon receipt, the message is decrypted, HMAC(Ki, {SeqNum, msg}) is computed and compared with the received value; the message is discarded if they differ. 3 Security Considerations 3.1 Authentication of Clients using Digest Authentication Digest Authentication does not provide a strong authentication mechanism, when compared to public key based mechanisms, for example. However, since it prevents chosen plaintext attacks, it is stronger than (e.g.) CRAM-MD5, which has been proposed for use with LDAP [10], POP and IMAP (see RFC 2195 [9]). It is intended to replace the much weaker and even more dangerous use of plaintext passwords; however, since it is still a password based mechanism it avoids some of the potential deployabilty issues with public-key, OTP or similar mechanisms. Leach & Newman Standards Track [Page 15] RFC 2831 Digest SASL Mechanism May 2000 Digest Authentication offers no confidentiality protection beyond protecting the actual password. All of the rest of the challenge and response are available to an eavesdropper, including the user's name and authentication realm. 3.2 Comparison of Digest with Plaintext Passwords The greatest threat to the type of transactions for which these protocols are used is network snooping. This kind of transaction might involve, for example, online access to a mail service whose use is restricted to paying subscribers. With plaintext password authentication an eavesdropper can obtain the password of the user. This not only permits him to access anything in the database, but, often worse, will permit access to anything else the user protects with the same password. 3.3 Replay Attacks Replay attacks are defeated if the client or the server chooses a fresh nonce for each authentication, as this specification requires. 3.4 Online dictionary attacks If the attacker can eavesdrop, then it can test any overheard nonce/response pairs against a (potentially very large) list of common words. Such a list is usually much smaller than the total number of possible passwords. The cost of computing the response for each password on the list is paid once for each challenge. The server can mitigate this attack by not allowing users to select passwords that are in a dictionary. 3.5 Offline dictionary attacks If the attacker can choose the challenge, then it can precompute the possible responses to that challenge for a list of common words. Such a list is usually much smaller than the total number of possible passwords. The cost of computing the response for each password on the list is paid just once. Offline dictionary attacks are defeated if the client chooses a fresh nonce for each authentication, as this specification requires. Leach & Newman Standards Track [Page 16] RFC 2831 Digest SASL Mechanism May 2000 3.6 Man in the Middle Digest authentication is vulnerable to "man in the middle" (MITM) attacks. Clearly, a MITM would present all the problems of eavesdropping. But it also offers some additional opportunities to the attacker. A possible man-in-the-middle attack would be to substitute a weaker qop scheme for the one(s) sent by the server; the server will not be able to detect this attack. For this reason, the client should always use the strongest scheme that it understands from the choices offered, and should never choose a scheme that does not meet its minimum requirements. 3.7 Chosen plaintext attacks A chosen plaintext attack is where a MITM or a malicious server can arbitrarily choose the challenge that the client will use to compute the response. The ability to choose the challenge is known to make cryptanalysis much easier [8]. However, Digest does not permit the attack to choose the challenge as long as the client chooses a fresh nonce for each authentication, as this specification requires. 3.8 Spoofing by Counterfeit Servers If a user can be led to believe that she is connecting to a host containing information protected by a password she knows, when in fact she is connecting to a hostile server, then the hostile server can obtain challenge/response pairs where it was able to partly choose the challenge. There is no known way that this can be exploited. 3.9 Storing passwords Digest authentication requires that the authenticating agent (usually the server) store some data derived from the user's name and password in a "password file" associated with a given realm. Normally this might contain pairs consisting of username and H({ username-value, ":", realm-value, ":", passwd }), which is adequate to compute H(A1) as described above without directly exposing the user's password. The security implications of this are that if this password file is compromised, then an attacker gains immediate access to documents on the server using this realm. Unlike, say a standard UNIX password file, this information need not be decrypted in order to access documents in the server realm associated with this file. On the other Leach & Newman Standards Track [Page 17] RFC 2831 Digest SASL Mechanism May 2000 hand, decryption, or more likely a brute force attack, would be necessary to obtain the user's password. This is the reason that the realm is part of the digested data stored in the password file. It means that if one Digest authentication password file is compromised, it does not automatically compromise others with the same username and password (though it does expose them to brute force attack). There are two important security consequences of this. First the password file must be protected as if it contained plaintext passwords, because for the purpose of accessing documents in its realm, it effectively does. A second consequence of this is that the realm string should be unique among all realms that any single user is likely to use. In particular a realm string should include the name of the host doing the authentication. 3.10 Multiple realms Use of multiple realms may mean both that compromise of a the security database for a single realm does not compromise all security, and that there are more things to protect in order to keep the whole system secure. 3.11 Summary By modern cryptographic standards Digest Authentication is weak, compared to (say) public key based mechanisms. But for a large range of purposes it is valuable as a replacement for plaintext passwords. Its strength may vary depending on the implementation. 4 Example This example shows the use of the Digest SASL mechanism with the IMAP4 AUTHENTICATE command [RFC 2060]. In this example, "C:" and "S:" represent a line sent by the client or server respectively including a CRLF at the end. Linebreaks and indentation within a "C:" or "S:" are editorial and not part of the protocol. The password in this example was "secret". Note that the base64 encoding of the challenges and responses is part of the IMAP4 AUTHENTICATE command, not part of the Digest specification itself. S: * OK elwood.innosoft.com PMDF IMAP4rev1 V6.0-9 C: c CAPABILITY S: * CAPABILITY IMAP4 IMAP4rev1 ACL LITERAL+ NAMESPACE QUOTA UIDPLUS AUTH=CRAM-MD5 AUTH=DIGEST-MD5 AUTH=PLAIN S: c OK Completed Leach & Newman Standards Track [Page 18] RFC 2831 Digest SASL Mechanism May 2000 C: a AUTHENTICATE DIGEST-MD5 S: + cmVhbG09ImVsd29vZC5pbm5vc29mdC5jb20iLG5vbmNlPSJPQTZNRzl0 RVFHbTJoaCIscW9wPSJhdXRoIixhbGdvcml0aG09bWQ1LXNlc3MsY2hh cnNldD11dGYtOA== C: Y2hhcnNldD11dGYtOCx1c2VybmFtZT0iY2hyaXMiLHJlYWxtPSJlbHdvb2 QuaW5ub3NvZnQuY29tIixub25jZT0iT0E2TUc5dEVRR20yaGgiLG5jPTAw MDAwMDAxLGNub25jZT0iT0E2TUhYaDZWcVRyUmsiLGRpZ2VzdC11cmk9Im ltYXAvZWx3b29kLmlubm9zb2Z0LmNvbSIscmVzcG9uc2U9ZDM4OGRhZDkw ZDRiYmQ3NjBhMTUyMzIxZjIxNDNhZjcscW9wPWF1dGg= S: + cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA== C: S: a OK User logged in --- The base64-decoded version of the SASL exchange is: S: realm="elwood.innosoft.com",nonce="OA6MG9tEQGm2hh",qop="auth", algorithm=md5-sess,charset=utf-8 C: charset=utf-8,username="chris",realm="elwood.innosoft.com", nonce="OA6MG9tEQGm2hh",nc=00000001,cnonce="OA6MHXh6VqTrRk", digest-uri="imap/elwood.innosoft.com", response=d388dad90d4bbd760a152321f2143af7,qop=auth S: rspauth=ea40f60335c427b5527b84dbabcdfffd The password in this example was "secret". This example shows the use of the Digest SASL mechanism with the ACAP, using the same notational conventions and password as in the previous example. Note that ACAP does not base64 encode and uses fewer round trips that IMAP4. S: * ACAP (IMPLEMENTATION "Test ACAP server") (SASL "CRAM-MD5" "DIGEST-MD5" "PLAIN") C: a AUTHENTICATE "DIGEST-MD5" S: + {94} S: realm="elwood.innosoft.com",nonce="OA9BSXrbuRhWay",qop="auth", algorithm=md5-sess,charset=utf-8 C: {206} C: charset=utf-8,username="chris",realm="elwood.innosoft.com", nonce="OA9BSXrbuRhWay",nc=00000001,cnonce="OA9BSuZWMSpW8m", digest-uri="acap/elwood.innosoft.com", response=6084c6db3fede7352c551284490fd0fc,qop=auth S: a OK (SASL {40} S: rspauth=2f0b3d7c3c2e486600ef710726aa2eae) "AUTHENTICATE Completed" --- Leach & Newman Standards Track [Page 19] RFC 2831 Digest SASL Mechanism May 2000 The server uses the values of all the directives, plus knowledge of the users password (or the hash of the user's name, server's realm and the user's password) to verify the computations above. If they check, then the user has authenticated. 5 References [Digest] Franks, J., et al., "HTTP Authentication: Basic and Digest Access Authentication", RFC 2617, June 1999. [ISO-8859] ISO-8859. International Standard--Information Processing-- 8-bit Single-Byte Coded Graphic Character Sets -- Part 1: Latin alphabet No. 1, ISO-8859-1:1987. Part 2: Latin alphabet No. 2, ISO-8859-2, 1987. Part 3: Latin alphabet No. 3, ISO-8859-3, 1988. Part 4: Latin alphabet No. 4, ISO-8859-4, 1988. Part 5: Latin/Cyrillic alphabet, ISO-8859-5, 1988. Part 6: Latin/Arabic alphabet, ISO-8859-6, 1987. Part 7: Latin/Greek alphabet, ISO-8859-7, 1987. Part 8: Latin/Hebrew alphabet, ISO-8859-8, 1988. Part 9: Latin alphabet No. 5, ISO-8859-9, 1990. [RFC 822] Crocker, D., "Standard for The Format of ARPA Internet Text Messages," STD 11, RFC 822, August 1982. [RFC 1321] Rivest, R., "The MD5 Message-Digest Algorithm", RFC 1321, April 1992. [RFC 2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. [RFC 2052] Gulbrandsen, A. and P. Vixie, "A DNS RR for specifying the location of services (DNS SRV)", RFC 2052, October 1996. [RFC 2060] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [RFC 2104] Krawczyk, H., Bellare, M. and R. Canetti, "HMAC: Keyed- Hashing for Message Authentication", RFC 2104, February 1997. [RFC 2195] Klensin, J., Catoe, R. and P. Krumviede, "IMAP/POP AUTHorize Extension for Simple Challenge/Response", RFC 2195, September 1997. Leach & Newman Standards Track [Page 20] RFC 2831 Digest SASL Mechanism May 2000 [RFC 2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC 2222] Myers, J., "Simple Authentication and Security Layer (SASL)", RFC 2222, October 1997. [USASCII] US-ASCII. Coded Character Set - 7-Bit American Standard Code for Information Interchange. Standard ANSI X3.4-1986, ANSI, 1986. 6 Authors' Addresses Paul Leach Microsoft 1 Microsoft Way Redmond, WA 98052 EMail: paulle@microsoft.com Chris Newman Innosoft International, Inc. 1050 Lakes Drive West Covina, CA 91790 USA EMail: chris.newman@innosoft.com 7 ABNF What follows is the definition of the notation as is used in the HTTP/1.1 specification (RFC 2616) and the HTTP authentication specification (RFC 2617); it is reproduced here for ease of reference. Since it is intended that a single Digest implementation can support both HTTP and SASL-based protocols, the same notation is used in both to facilitate comparison and prevention of unwanted differences. Since it is cut-and-paste from the HTTP specifications, not all productions may be used in this specification. It is also not quite legal ABNF; again, the errors were copied from the HTTP specifications. 7.1 Augmented BNF All of the mechanisms specified in this document are described in both prose and an augmented Backus-Naur Form (BNF) similar to that used by RFC 822 [RFC 822]. Implementers will need to be familiar with the notation in order to understand this specification. Leach & Newman Standards Track [Page 21] RFC 2831 Digest SASL Mechanism May 2000 The augmented BNF includes the following constructs: name = definition The name of a rule is simply the name itself (without any enclosing "<" and ">") and is separated from its definition by the equal "=" character. White space is only significant in that indentation of continuation lines is used to indicate a rule definition that spans more than one line. Certain basic rules are in uppercase, such as SP, LWS, HT, CRLF, DIGIT, ALPHA, etc. Angle brackets are used within definitions whenever their presence will facilitate discerning the use of rule names. "literal" Quotation marks surround literal text. Unless stated otherwise, the text is case-insensitive. rule1 | rule2 Elements separated by a bar ("|") are alternatives, e.g., "yes | no" will accept yes or no. (rule1 rule2) Elements enclosed in parentheses are treated as a single element. Thus, "(elem (foo | bar) elem)" allows the token sequences "elem foo elem" and "elem bar elem". *rule The character "*" preceding an element indicates repetition. The full form is "*element" indicating at least and at most occurrences of element. Default values are 0 and infinity so that "*(element)" allows any number, including zero; "1*element" requires at least one; and "1*2element" allows one or two. [rule] Square brackets enclose optional elements; "[foo bar]" is equivalent to "*1(foo bar)". N rule Specific repetition: "(element)" is equivalent to "*(element)"; that is, exactly occurrences of (element). Thus 2DIGIT is a 2-digit number, and 3ALPHA is a string of three alphabetic characters. #rule A construct "#" is defined, similar to "*", for defining lists of elements. The full form is "#element" indicating at least and at most elements, each separated by one or more commas (",") and OPTIONAL linear white space (LWS). This makes the usual form of lists very easy; a rule such as Leach & Newman Standards Track [Page 22] RFC 2831 Digest SASL Mechanism May 2000 ( *LWS element *( *LWS "," *LWS element )) can be shown as 1#element Wherever this construct is used, null elements are allowed, but do not contribute to the count of elements present. That is, "(element), , (element) " is permitted, but counts as only two elements. Therefore, where at least one element is required, at least one non-null element MUST be present. Default values are 0 and infinity so that "#element" allows any number, including zero; "1#element" requires at least one; and "1#2element" allows one or two. ; comment A semi-colon, set off some distance to the right of rule text, starts a comment that continues to the end of line. This is a simple way of including useful notes in parallel with the specifications. implied *LWS The grammar described by this specification is word-based. Except where noted otherwise, linear white space (LWS) can be included between any two adjacent words (token or quoted-string), and between adjacent words and separators, without changing the interpretation of a field. At least one delimiter (LWS and/or separators) MUST exist between any two tokens (for the definition of "token" below), since they would otherwise be interpreted as a single token. 7.2 Basic Rules The following rules are used throughout this specification to describe basic parsing constructs. The US-ASCII coded character set is defined by ANSI X3.4-1986 [USASCII]. OCTET = CHAR = UPALPHA = LOALPHA = ALPHA = UPALPHA | LOALPHA DIGIT = CTL = CR = LF = SP = HT = <"> = CRLF = CR LF Leach & Newman Standards Track [Page 23] RFC 2831 Digest SASL Mechanism May 2000 All linear white space, including folding, has the same semantics as SP. A recipient MAY replace any linear white space with a single SP before interpreting the field value or forwarding the message downstream. LWS = [CRLF] 1*( SP | HT ) The TEXT rule is only used for descriptive field contents and values that are not intended to be interpreted by the message parser. Words of *TEXT MAY contain characters from character sets other than ISO-8859-1 [ISO 8859] only when encoded according to the rules of RFC 2047 [RFC 2047]. TEXT = A CRLF is allowed in the definition of TEXT only as part of a header field continuation. It is expected that the folding LWS will be replaced with a single SP before interpretation of the TEXT value. Hexadecimal numeric characters are used in several protocol elements. HEX = "A" | "B" | "C" | "D" | "E" | "F" | "a" | "b" | "c" | "d" | "e" | "f" | DIGIT Many HTTP/1.1 header field values consist of words separated by LWS or special characters. These special characters MUST be in a quoted string to be used within a parameter value. token = 1* separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT A string of text is parsed as a single word if it is quoted using double-quote marks. quoted-string = ( <"> qdstr-val <"> ) qdstr-val = *( qdtext | quoted-pair ) qdtext = > Note that LWS is NOT implicit between the double-quote marks (<">) surrounding a qdstr-val and the qdstr-val; any LWS will be considered part of the qdstr-val. This is also the case for quotation marks surrounding any other construct. Leach & Newman Standards Track [Page 24] RFC 2831 Digest SASL Mechanism May 2000 The backslash character ("\") MAY be used as a single-character quoting mechanism only within qdstr-val and comment constructs. quoted-pair = "\" CHAR The value of this construct is CHAR. Note that an effect of this rule is that backslash must be quoted. 8 Sample Code The sample implementation in [Digest] also applies to DIGEST-MD5. The following code implements the conversion from UTF-8 to 8859-1 if necessary. /* if the string is entirely in the 8859-1 subset of UTF-8, then * translate to 8859-1 prior to MD5 */ void MD5_UTF8_8859_1(MD5_CTX *ctx, const unsigned char *base, int len) { const unsigned char *scan, *end; unsigned char cbuf; end = base + len; for (scan = base; scan < end; ++scan) { if (*scan > 0xC3) break; /* abort if outside 8859-1 */ if (*scan >= 0xC0 && *scan <= 0xC3) { if (++scan == end || *scan < 0x80 || *scan > 0xBF) break; } } /* if we found a character outside 8859-1, don't alter string */ if (scan < end) { MD5Update(ctx, base, len); return; } /* convert to 8859-1 prior to applying hash */ do { for (scan = base; scan < end && *scan < 0xC0; ++scan) ; if (scan != base) MD5Update(ctx, base, scan - base); if (scan + 1 >= end) break; cbuf = ((scan[0] & 0x3) << 6) | (scan[1] & 0x3f); MD5Update(ctx, &cbuf, 1); Leach & Newman Standards Track [Page 25] RFC 2831 Digest SASL Mechanism May 2000 base = scan + 2; } while (base < end); } Leach & Newman Standards Track [Page 26] RFC 2831 Digest SASL Mechanism May 2000 9 Full Copyright Statement Copyright (C) The Internet Society (2000). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Leach & Newman Standards Track [Page 27] ================================================ FILE: docs/rfcs/rfc2971.IMAP4_ID_extension.txt ================================================ Network Working Group T. Showalter Request for Comments: 2971 Mirapoint, Inc. Category: Standards Track October 2000 IMAP4 ID extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2000). All Rights Reserved. Abstract The ID extension to the Internet Message Access Protocol - Version 4rev1 (IMAP4rev1) protocol allows the server and client to exchange identification information on their implementation in order to make bug reports and usage statistics more complete. 1. Introduction The IMAP4rev1 protocol described in [IMAP4rev1] provides a method for accessing remote mail stores, but it provides no facility to advertise what program a client or server uses to provide service. This makes it difficult for implementors to get complete bug reports from users, as it is frequently difficult to know what client or server is in use. Additionally, some sites may wish to assemble usage statistics based on what clients are used, but in an an environment where users are permitted to obtain and maintain their own clients this is difficult to accomplish. The ID command provides a facility to advertise information on what programs are being used along with contact information (should bugs ever occur). Showalter Standards Track [Page 1] RFC 2971 IMAP4 ID extension October 2000 2. Conventions Used in this Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. The conventions used in this document are the same as specified in [IMAP4rev1]. In examples, "C:" and "S:" indicate lines sent by the client and server respectively. Line breaks have been inserted for readability. 3. Specification The sole purpose of the ID extension is to enable clients and servers to exchange information on their implementations for the purposes of statistical analysis and problem determination. This information is be submitted to a server by any client wishing to provide information for statistical purposes, provided the server advertises its willingness to take the information with the atom "ID" included in the list of capabilities returned by the CAPABILITY command. Implementations MUST NOT make operational changes based on the data sent as part of the ID command or response. The ID command is for human consumption only, and is not to be used in improving the performance of clients or servers. This includes, but is not limited to, the following: Servers MUST NOT attempt to work around client bugs by using information from the ID command. Clients MUST NOT attempt to work around server bugs based on the ID response. Servers MUST NOT provide features to a client or otherwise optimize for a particular client by using information from the ID command. Clients MUST NOT provide features to a server or otherwise optimize for a particular server based on the ID response. Servers MUST NOT deny access to or refuse service for a client based on information from the ID command. Clients MUST NOT refuse to operate or limit their operation with a server based on the ID response. Showalter Standards Track [Page 2] RFC 2971 IMAP4 ID extension October 2000 Rationale: It is imperative that this extension not supplant IMAP's CAPABILITY mechanism with a ad-hoc approach where implementations guess each other's features based on who they claim to be. Implementations MUST NOT send false information in an ID command. Implementations MAY send less information than they have available or no information at all. Such behavior may be useful to preserve user privacy. See Security Considerations, section 7. 3.1. ID Command Arguments: client parameter list or NIL Responses: OPTIONAL untagged response: ID Result: OK identification information accepted BAD command unknown or arguments invalid Implementation identification information is sent by the client with the ID command. This command is valid in any state. The information sent is in the form of a list of field/value pairs. Fields are permitted to be any IMAP4 string, and values are permitted to be any IMAP4 string or NIL. A value of NIL indicates that the client can not or will not specify this information. The client may also send NIL instead of the list, indicating that it wants to send no information, but would still accept a server response. The available fields are defined in section 3.3. Example: C: a023 ID ("name" "sodr" "version" "19.34" "vendor" "Pink Floyd Music Limited") S: * ID NIL S: a023 OK ID completed 3.2. ID Response Contents: server parameter list In response to an ID command issued by the client, the server replies with a tagged response containing information on its implementation. The format is the same as the client list. Showalter Standards Track [Page 3] RFC 2971 IMAP4 ID extension October 2000 Example: C: a042 ID NIL S: * ID ("name" "Cyrus" "version" "1.5" "os" "sunos" "os-version" "5.5" "support-url" "mailto:cyrus-bugs+@andrew.cmu.edu") S: a042 OK ID command completed A server MUST send a tagged ID response to an ID command. However, a server MAY send NIL in place of the list. 3.3. Defined Field Values Any string may be sent as a field, but the following are defined to describe certain values that might be sent. Implementations are free to send none, any, or all of these. Strings are not case-sensitive. Field strings MUST NOT be longer than 30 octets. Value strings MUST NOT be longer than 1024 octets. Implementations MUST NOT send more than 30 field-value pairs. name Name of the program version Version number of the program os Name of the operating system os-version Version of the operating system vendor Vendor of the client/server support-url URL to contact for support address Postal address of contact/vendor date Date program was released, specified as a date-time in IMAP4rev1 command Command used to start the program arguments Arguments supplied on the command line, if any if any environment Description of environment, i.e., UNIX environment variables or Windows registry settings Implementations MUST NOT use contact information to submit automatic bug reports. Implementations may include information from an ID response in a report automatically prepared, but are prohibited from sending the report without user authorization. It is preferable to find the name and version of the underlying operating system at runtime in cases where this is possible. Information sent via an ID response may violate user privacy. See Security Considerations, section 7. Implementations MUST NOT send the same field name more than once. Showalter Standards Track [Page 4] RFC 2971 IMAP4 ID extension October 2000 4. Formal Syntax This syntax is intended to augment the grammar specified in [IMAP4rev1] in order to provide for the ID command. This specification uses the augmented Backus-Naur Form (BNF) notation as used in [IMAP4rev1]. command_any ::= "CAPABILITY" / "LOGOUT" / "NOOP" / x_command / id ;; adds id command to command_any in [IMAP4rev1] id ::= "ID" SPACE id_params_list id_response ::= "ID" SPACE id_params_list id_params_list ::= "(" #(string SPACE nstring) ")" / nil ;; list of field value pairs response_data ::= "*" SPACE (resp_cond_state / resp_cond_bye / mailbox_data / message_data / capability_data / id_response) 5. Use of the ID extension with Firewalls and Other Intermediaries There exist proxies, firewalls, and other intermediary systems that can intercept an IMAP session and make changes to the data exchanged in the session. Such intermediaries are not anticipated by the IMAP4 protocol design and are not within the scope of the IMAP4 standard. However, in order for the ID command to be useful in the presence of such intermediaries, those intermediaries need to take special note of the ID command and response. In particular, if an intermediary changes any part of the IMAP session it must also change the ID command to advertise its presence. A firewall MAY act to block transmission of specific information fields in the ID command and response that it believes reveal information that could expose a security vulnerability. However, a firewall SHOULD NOT disable the extension, when present, entirely, and SHOULD NOT unconditionally remove either the client or server list. Finally, it should be noted that a firewall, when handling a CAPABILITY response, MUST NOT allow the names of extensions to be returned to the client that the firewall has no knowledge of. Showalter Standards Track [Page 5] RFC 2971 IMAP4 ID extension October 2000 6. References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, March 1997. [IMAP4rev1] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, October 1996. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822, August 1982. 7. Security Considerations This extension has the danger of violating the privacy of users if misused. Clients and servers should notify users that they implement and enable the ID command. It is highly desirable that implementations provide a method of disabling ID support, perhaps by not sending ID at all, or by sending NIL as the argument to the ID command or response. Implementors must exercise extreme care in adding fields sent as part of an ID command or response. Some fields, including a processor ID number, Ethernet address, or other unique (or mostly unique) identifier allow tracking of users in ways that violate user privacy expectations. Having implementation information of a given client or server may make it easier for an attacker to gain unauthorized access due to security holes. Since this command includes arbitrary data and does not require the user to authenticate, server implementations are cautioned to guard against an attacker sending arbitrary garbage data in order to fill up the ID log. In particular, if a server naively logs each ID command to disk without inspecting it, an attacker can simply fire up thousands of connections and send a few kilobytes of random data. Servers have to guard against this. Methods include truncating abnormally large responses; collating responses by storing only a single copy, then keeping a counter of the number of times that response has been seen; keeping only particularly interesting parts of responses; and only logging responses of users who actually log in. Security is affected by firewalls which modify the IMAP protocol stream; see section 5, Use of the ID Extension with Firewalls and Other Intermediaries, for more information. Showalter Standards Track [Page 6] RFC 2971 IMAP4 ID extension October 2000 8. Author's Address Tim Showalter Mirapoint, Inc. 909 Hermosa Ct. Sunnyvale, CA 94095 EMail: tjs@mirapoint.com Showalter Standards Track [Page 7] RFC 2971 IMAP4 ID extension October 2000 9. Full Copyright Statement Copyright (C) The Internet Society (2000). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Showalter Standards Track [Page 8] ================================================ FILE: docs/rfcs/rfc3028.Sieve_Mail_filtering_language.txt ================================================ Network Working Group T. Showalter Request for Comments: 3028 Mirapoint, Inc. Category: Standards Track January 2001 Sieve: A Mail Filtering Language Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2001). All Rights Reserved. Abstract This document describes a language for filtering e-mail messages at time of final delivery. It is designed to be implementable on either a mail client or mail server. It is meant to be extensible, simple, and independent of access protocol, mail architecture, and operating system. It is suitable for running on a mail server where users may not be allowed to execute arbitrary programs, such as on black box Internet Message Access Protocol (IMAP) servers, as it has no variables, loops, or ability to shell out to external programs. Table of Contents 1. Introduction ........................................... 3 1.1. Conventions Used in This Document ..................... 4 1.2. Example mail messages ................................. 4 2. Design ................................................. 5 2.1. Form of the Language .................................. 5 2.2. Whitespace ............................................ 5 2.3. Comments .............................................. 6 2.4. Literal Data .......................................... 6 2.4.1. Numbers ............................................... 6 2.4.2. Strings ............................................... 7 2.4.2.1. String Lists .......................................... 7 2.4.2.2. Headers ............................................... 8 2.4.2.3. Addresses ............................................. 8 2.4.2.4. MIME Parts ............................................ 9 2.5. Tests ................................................. 9 2.5.1. Test Lists ............................................ 9 Showalter Standards Track [Page 1] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.6. Arguments ............................................. 9 2.6.1. Positional Arguments .................................. 9 2.6.2. Tagged Arguments ...................................... 10 2.6.3. Optional Arguments .................................... 10 2.6.4. Types of Arguments .................................... 10 2.7. String Comparison ..................................... 11 2.7.1. Match Type ............................................ 11 2.7.2. Comparisons Across Character Sets ..................... 12 2.7.3. Comparators ........................................... 12 2.7.4. Comparisons Against Addresses ......................... 13 2.8. Blocks ................................................ 14 2.9. Commands .............................................. 14 2.10. Evaluation ............................................ 15 2.10.1. Action Interaction .................................... 15 2.10.2. Implicit Keep ......................................... 15 2.10.3. Message Uniqueness in a Mailbox ....................... 15 2.10.4. Limits on Numbers of Actions .......................... 16 2.10.5. Extensions and Optional Features ...................... 16 2.10.6. Errors ................................................ 17 2.10.7. Limits on Execution ................................... 17 3. Control Commands ....................................... 17 3.1. Control Structure If .................................. 18 3.2. Control Structure Require ............................. 19 3.3. Control Structure Stop ................................ 19 4. Action Commands ........................................ 19 4.1. Action reject ......................................... 20 4.2. Action fileinto ....................................... 20 4.3. Action redirect ....................................... 21 4.4. Action keep ........................................... 21 4.5. Action discard ........................................ 22 5. Test Commands .......................................... 22 5.1. Test address .......................................... 23 5.2. Test allof ............................................ 23 5.3. Test anyof ............................................ 24 5.4. Test envelope ......................................... 24 5.5. Test exists ........................................... 25 5.6. Test false ............................................ 25 5.7. Test header ........................................... 25 5.8. Test not .............................................. 26 5.9. Test size ............................................. 26 5.10. Test true ............................................. 26 6. Extensibility .......................................... 26 6.1. Capability String ..................................... 27 6.2. IANA Considerations ................................... 28 6.2.1. Template for Capability Registrations ................. 28 6.2.2. Initial Capability Registrations ...................... 28 6.3. Capability Transport .................................. 29 7. Transmission ........................................... 29 Showalter Standards Track [Page 2] RFC 3028 Sieve: A Mail Filtering Language January 2001 8. Parsing ................................................ 30 8.1. Lexical Tokens ........................................ 30 8.2. Grammar ............................................... 31 9. Extended Example ....................................... 32 10. Security Considerations ................................ 34 11. Acknowledgments ........................................ 34 12. Author's Address ....................................... 34 13. References ............................................. 34 14. Full Copyright Statement ............................... 36 1. Introduction This memo documents a language that can be used to create filters for electronic mail. It is not tied to any particular operating system or mail architecture. It requires the use of [IMAIL]-compliant messages, but should otherwise generalize to many systems. The language is powerful enough to be useful but limited in order to allow for a safe server-side filtering system. The intention is to make it impossible for users to do anything more complex (and dangerous) than write simple mail filters, along with facilitating the use of GUIs for filter creation and manipulation. The language is not Turing-complete: it provides no way to write a loop or a function and variables are not provided. Scripts written in Sieve are executed during final delivery, when the message is moved to the user-accessible mailbox. In systems where the MTA does final delivery, such as traditional Unix mail, it is reasonable to sort when the MTA deposits mail into the user's mailbox. There are a number of reasons to use a filtering system. Mail traffic for most users has been increasing due to increased usage of e-mail, the emergence of unsolicited email as a form of advertising, and increased usage of mailing lists. Experience at Carnegie Mellon has shown that if a filtering system is made available to users, many will make use of it in order to file messages from specific users or mailing lists. However, many others did not make use of the Andrew system's FLAMES filtering language [FLAMES] due to difficulty in setting it up. Because of the expectation that users will make use of filtering if it is offered and easy to use, this language has been made simple enough to allow many users to make use of it, but rich enough that it can be used productively. However, it is expected that GUI-based editors will be the preferred way of editing filters for a large number of users. Showalter Standards Track [Page 3] RFC 3028 Sieve: A Mail Filtering Language January 2001 1.1. Conventions Used in This Document In the sections of this document that discuss the requirements of various keywords and operators, the following conventions have been adopted. The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in [KEYWORDS]. Each section on a command (test, action, or control structure) has a line labeled "Syntax:". This line describes the syntax of the command, including its name and its arguments. Required arguments are listed inside angle brackets ("<" and ">"). Optional arguments are listed inside square brackets ("[" and "]"). Each argument is followed by its type, so "" represents an argument called "key" that is a string. Literal strings are represented with double-quoted strings. Alternatives are separated with slashes, and parenthesis are used for grouping, similar to [ABNF]. In the "Syntax" line, there are three special pieces of syntax that are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, respectively. The formal grammar for these commands in section 10 and is the authoritative reference on how to construct commands, but the formal grammar does not specify the order, semantics, number or types of arguments to commands, nor the legal command names. The intent is to allow for extension without changing the grammar. 1.2. Example mail messages The following mail messages will be used throughout this document in examples. Message A ----------------------------------------------------------- Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) From: coyote@desert.example.org To: roadrunner@acme.example.com Subject: I have a present for you Look, I'm sorry about the whole anvil thing, and I really didn't mean to try and drop it on you from the top of the cliff. I want to try to make it up to you. I've got some great birdseed over here at my place--top of the line Showalter Standards Track [Page 4] RFC 3028 Sieve: A Mail Filtering Language January 2001 stuff--and if you come by, I'll have it all wrapped up for you. I'm really sorry for all the problems I've caused for you over the years, but I know we can work this out. -- Wile E. Coyote "Super Genius" coyote@desert.example.org ----------------------------------------------------------- Message B ----------------------------------------------------------- From: youcouldberich!@reply-by-postal-mail.invalid Sender: b1ff@de.res.example.com To: rube@landru.example.edu Date: Mon, 31 Mar 1997 18:26:10 -0800 Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! ----------------------------------------------------------- 2. Design 2.1. Form of the Language The language consists of a set of commands. Each command consists of a set of tokens delimited by whitespace. The command identifier is the first token and it is followed by zero or more argument tokens. Arguments may be literal data, tags, blocks of commands, or test commands. The language is represented in UTF-8, as specified in [UTF-8]. Tokens in the ASCII range are considered case-insensitive. 2.2. Whitespace Whitespace is used to separate tokens. Whitespace is made up of tabs, newlines (CRLF, never just CR or LF), and the space character. The amount of whitespace used is not significant. Showalter Standards Track [Page 5] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.3. Comments Two types of comments are offered. Comments are semantically equivalent to whitespace and can be used anyplace that whitespace is (with one exception in multi-line strings, as described in the grammar). Hash comments begin with a "#" character that is not contained within a string and continue until the next CRLF. Example: if size :over 100K { # this is a comment discard; } Bracketed comments begin with the token "/*" and end with "*/" outside of a string. Bracketed comments may span multiple lines. Bracketed comments do not nest. Example: if size :over 100K { /* this is a comment this is still a comment */ discard /* this is a comment */ ; } 2.4. Literal Data Literal data means data that is not executed, merely evaluated "as is", to be used as arguments to commands. Literal data is limited to numbers and strings. 2.4.1. Numbers Numbers are given as ordinary decimal numbers. However, those numbers that have a tendency to be fairly large, such as message sizes, MAY have a "K", "M", or "G" appended to indicate a multiple of a power of two. To be comparable with the power-of-two-based versions of SI units that computers frequently use, K specifies kibi-, or 1,024 (2^10) times the value of the number; M specifies mebi-, or 1,048,576 (2^20) times the value of the number; and G specifies tebi-, or 1,073,741,824 (2^30) times the value of the number [BINARY-SI]. Implementations MUST provide 31 bits of magnitude in numbers, but MAY provide more. Only positive integers are permitted by this specification. Showalter Standards Track [Page 6] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.4.2. Strings Scripts involve large numbers of strings as they are used for pattern matching, addresses, textual bodies, etc. Typically, short quoted strings suffice for most uses, but a more convenient form is provided for longer strings such as bodies of messages. A quoted string starts and ends with a single double quote (the <"> character, ASCII 34). A backslash ("\", ASCII 92) inside of a quoted string is followed by either another backslash or a double quote. This two-character sequence represents a single backslash or double- quote within the string, respectively. No other characters should be escaped with a single backslash. An undefined escape sequence (such as "\a" in a context where "a" has no special meaning) is interpreted as if there were no backslash (in this case, "\a" is just "a"). Non-printing characters such as tabs, CR and LF, and control characters are permitted in quoted strings. Quoted strings MAY span multiple lines. NUL (ASCII 0) is not allowed in strings. For entering larger amounts of text, such as an email message, a multi-line form is allowed. It starts with the keyword "text:", followed by a CRLF, and ends with the sequence of a CRLF, a single period, and another CRLF. In order to allow the message to contain lines with a single-dot, lines are dot-stuffed. That is, when composing a message body, an extra `.' is added before each line which begins with a `.'. When the server interprets the script, these extra dots are removed. Note that a line that begins with a dot followed by a non-dot character is not interpreted dot-stuffed; that is, ".foo" is interpreted as ".foo". However, because this is potentially ambiguous, scripts SHOULD be properly dot-stuffed so such lines do not appear. Note that a hashed comment or whitespace may occur in between the "text:" and the CRLF, but not within the string itself. Bracketed comments are not allowed here. 2.4.2.1. String Lists When matching patterns, it is frequently convenient to match against groups of strings instead of single strings. For this reason, a list of strings is allowed in many tests, implying that if the test is true using any one of the strings, then the test is true. Implementations are encouraged to use short-circuit evaluation in these cases. Showalter Standards Track [Page 7] RFC 3028 Sieve: A Mail Filtering Language January 2001 For instance, the test `header :contains ["To", "Cc"] ["me@example.com", "me00@landru.example.edu"]' is true if either the To header or Cc header of the input message contains either of the e-mail addresses "me@example.com" or "me00@landru.example.edu". Conversely, in any case where a list of strings is appropriate, a single string is allowed without being a member of a list: it is equivalent to a list with a single member. This means that the test `exists "To"' is equivalent to the test `exists ["To"]'. 2.4.2.2. Headers Headers are a subset of strings. In the Internet Message Specification [IMAIL] [RFC1123], each header line is allowed to have whitespace nearly anywhere in the line, including after the field name and before the subsequent colon. Extra spaces between the header name and the ":" in a header field are ignored. A header name never contains a colon. The "From" header refers to a line beginning "From:" (or "From :", etc.). No header will match the string "From:" due to the trailing colon. Folding of long header lines (as described in [IMAIL] 3.4.8) is removed prior to interpretation of the data. The folding syntax (the CRLF that ends a line plus any leading whitespace at the beginning of the next line that indicates folding) are interpreted as if they were a single space. 2.4.2.3. Addresses A number of commands call for email addresses, which are also a subset of strings. When these addresses are used in outbound contexts, addresses must be compliant with [IMAIL], but are further constrained. Using the symbols defined in [IMAIL], section 6.1, the syntax of an address is: sieve-address = addr-spec ; simple address / phrase "<" addr-spec ">" ; name & addr-spec That is, routes and group syntax are not permitted. If multiple addresses are required, use a string list. Named groups are not used here. Implementations MUST ensure that the addresses are syntactically valid, but need not ensure that they actually identify an email recipient. Showalter Standards Track [Page 8] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.4.2.4. MIME Parts In a few places, [MIME] body parts are represented as strings. These parts include MIME headers and the body. This provides a way of embedding typed data within a Sieve script so that, among other things, character sets other than UTF-8 can be used for output messages. 2.5. Tests Tests are given as arguments to commands in order to control their actions. In this document, tests are given to if/elsif/else to decide which block of code is run. Tests MUST NOT have side effects. That is, a test cannot affect the state of the filter or message. No tests in this specification have side effects, and side effects are forbidden in extension tests as well. The rationale for this is that tests with side effects impair readability and maintainability and are difficult to represent in a graphic interface for generating scripts. Side effects are confined to actions where they are clearer. 2.5.1. Test Lists Some tests ("allof" and "anyof", which implement logical "and" and logical "or", respectively) may require more than a single test as an argument. The test-list syntax element provides a way of grouping tests. Example: if anyof (not exists ["From", "Date"], header :contains "from" "fool@example.edu") { discard; } 2.6. Arguments In order to specify what to do, most commands take arguments. There are three types of arguments: positional, tagged, and optional. 2.6.1. Positional Arguments Positional arguments are given to a command which discerns their meaning based on their order. When a command takes positional arguments, all positional arguments must be supplied and must be in the order prescribed. Showalter Standards Track [Page 9] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.6.2. Tagged Arguments This document provides for tagged arguments in the style of CommonLISP. These are also similar to flags given to commands in most command-line systems. A tagged argument is an argument for a command that begins with ":" followed by a tag naming the argument, such as ":contains". This argument means that zero or more of the next tokens have some particular meaning depending on the argument. These next tokens may be numbers or strings but they are never blocks. Tagged arguments are similar to positional arguments, except that instead of the meaning being derived from the command, it is derived from the tag. Tagged arguments must appear before positional arguments, but they may appear in any order with other tagged arguments. For simplicity of the specification, this is not expressed in the syntax definitions with commands, but they still may be reordered arbitrarily provided they appear before positional arguments. Tagged arguments may be mixed with optional arguments. To simplify this specification, tagged arguments SHOULD NOT take tagged arguments as arguments. 2.6.3. Optional Arguments Optional arguments are exactly like tagged arguments except that they may be left out, in which case a default value is implied. Because optional arguments tend to result in shorter scripts, they have been used far more than tagged arguments. One particularly noteworthy case is the ":comparator" argument, which allows the user to specify which [ACAP] comparator will be used to compare two strings, since different languages may impose different orderings on UTF-8 [UTF-8] characters. 2.6.4. Types of Arguments Abstractly, arguments may be literal data, tests, or blocks of commands. In this way, an "if" control structure is merely a command that happens to take a test and a block as arguments and may execute the block of code. However, this abstraction is ambiguous from a parsing standpoint. The grammar in section 9.2 presents a parsable version of this: Arguments are string-lists, numbers, and tags, which may be followed Showalter Standards Track [Page 10] RFC 3028 Sieve: A Mail Filtering Language January 2001 by a test or a test-list, which may be followed by a block of commands. No more than one test or test list, nor more than one block of commands, may be used, and commands that end with blocks of commands do not end with semicolons. 2.7. String Comparison When matching one string against another, there are a number of ways of performing the match operation. These are accomplished with three types of matches: an exact match, a substring match, and a wildcard glob-style match. These are described below. In order to provide for matches between character sets and case insensitivity, Sieve borrows ACAP's comparator registry. However, when a string represents the name of a header, the comparator is never user-specified. Header comparisons are always done with the "i;ascii-casemap" operator, i.e., case-insensitive comparisons, because this is the way things are defined in the message specification [IMAIL]. 2.7.1. Match Type There are three match types describing the matching used in this specification: ":is", ":contains", and ":matches". Match type arguments are supplied to those commands which allow them to specify what kind of match is to be performed. These are used as tagged arguments to tests that perform string comparison. The ":contains" match type describes a substring match. If the value argument contains the key argument as a substring, the match is true. For instance, the string "frobnitzm" contains "frob" and "nit", but not "fbm". The null key ("") is contained in all values. The ":is" match type describes an absolute match; if the contents of the first string are absolutely the same as the contents of the second string, they match. Only the string "frobnitzm" is the string "frobnitzm". The null key ":is" and only ":is" the null value. The ":matches" version specifies a wildcard match using the characters "*" and "?". "*" matches zero or more characters, and "?" matches a single character. "?" and "*" may be escaped as "\\?" and "\\*" in strings to match against themselves. The first backslash escapes the second backslash; together, they escape the "*". This is awkward, but it is commonplace in several programming languages that use globs and regular expressions. Showalter Standards Track [Page 11] RFC 3028 Sieve: A Mail Filtering Language January 2001 In order to specify what type of match is supposed to happen, commands that support matching take optional tagged arguments ":matches", ":is", and ":contains". Commands default to using ":is" matching if no match type argument is supplied. Note that these modifiers may interact with comparators; in particular, some comparators are not suitable for matching with ":contains" or ":matches". It is an error to use a comparator with ":contains" or ":matches" that is not compatible with it. It is an error to give more than one of these arguments to a given command. For convenience, the "MATCH-TYPE" syntax element is defined here as follows: Syntax: ":is" / ":contains" / ":matches" 2.7.2. Comparisons Across Character Sets All Sieve scripts are represented in UTF-8, but messages may involve a number of character sets. In order for comparisons to work across character sets, implementations SHOULD implement the following behavior: Implementations decode header charsets to UTF-8. Two strings are considered equal if their UTF-8 representations are identical. Implementations should decode charsets represented in the forms specified by [MIME] for both message headers and bodies. Implementations must be capable of decoding US-ASCII, ISO-8859-1, the ASCII subset of ISO-8859-* character sets, and UTF-8. If implementations fail to support the above behavior, they MUST conform to the following: No two strings can be considered equal if one contains octets greater than 127. 2.7.3. Comparators In order to allow for language-independent, case-independent matches, the match type may be coupled with a comparator name. Comparators are described for [ACAP]; a registry is defined for ACAP, and this specification uses that registry. ACAP defines multiple comparator types. Only equality types are used in this specification. Showalter Standards Track [Page 12] RFC 3028 Sieve: A Mail Filtering Language January 2001 All implementations MUST support the "i;octet" comparator (simply compares octets) and the "i;ascii-casemap" comparator (which treats uppercase and lowercase characters in the ASCII subset of UTF-8 as the same). If left unspecified, the default is "i;ascii-casemap". Some comparators may not be usable with substring matches; that is, they may only work with ":is". It is an error to try and use a comparator with ":matches" or ":contains" that is not compatible with it. A comparator is specified by the ":comparator" option with commands that support matching. This option is followed by a string providing the name of the comparator to be used. For convenience, the syntax of a comparator is abbreviated to "COMPARATOR", and (repeated in several tests) is as follows: Syntax: ":comparator" So in this example, Example: if header :contains :comparator "i;octet" "Subject" "MAKE MONEY FAST" { discard; } would discard any message with subjects like "You can MAKE MONEY FAST", but not "You can Make Money Fast", since the comparator used is case-sensitive. Comparators other than i;octet and i;ascii-casemap must be declared with require, as they are extensions. If a comparator declared with require is not known, it is an error, and execution fails. If the comparator is not declared with require, it is also an error, even if the comparator is supported. (See 2.10.5.) Both ":matches" and ":contains" match types are compatible with the "i;octet" and "i;ascii-casemap" comparators and may be used with them. It is an error to give more than one of these arguments to a given command. 2.7.4. Comparisons Against Addresses Addresses are one of the most frequent things represented as strings. These are structured, and being able to compare against the local- part or the domain of an address is useful, so some tests that act Showalter Standards Track [Page 13] RFC 3028 Sieve: A Mail Filtering Language January 2001 exclusively on addresses take an additional optional argument that specifies what the test acts on. These optional arguments are ":localpart", ":domain", and ":all", which act on the local-part (left-side), the domain part (right- side), and the whole address. The kind of comparison done, such as whether or not the test done is case-insensitive, is specified as a comparator argument to the test. If an optional address-part is omitted, the default is ":all". It is an error to give more than one of these arguments to a given command. For convenience, the "ADDRESS-PART" syntax element is defined here as follows: Syntax: ":localpart" / ":domain" / ":all" 2.8. Blocks Blocks are sets of commands enclosed within curly braces. Blocks are supplied to commands so that the commands can implement control commands. A control structure is a command that happens to take a test and a block as one of its arguments; depending on the result of the test supplied as another argument, it runs the code in the block some number of times. With the commands supplied in this memo, there are no loops. The control structures supplied--if, elsif, and else--run a block either once or not at all. So there are two arguments, the test and the block. 2.9. Commands Sieve scripts are sequences of commands. Commands can take any of the tokens above as arguments, and arguments may be either tagged or positional arguments. Not all commands take all arguments. There are three kinds of commands: test commands, action commands, and control commands. The simplest is an action command. An action command is an identifier followed by zero or more arguments, terminated by a semicolon. Action commands do not take tests or blocks as arguments. Showalter Standards Track [Page 14] RFC 3028 Sieve: A Mail Filtering Language January 2001 A control command is similar, but it takes a test as an argument, and ends with a block instead of a semicolon. A test command is used as part of a control command. It is used to specify whether or not the block of code given to the control command is executed. 2.10. Evaluation 2.10.1. Action Interaction Some actions cannot be used with other actions because the result would be absurd. These restrictions are noted throughout this memo. Extension actions MUST state how they interact with actions defined in this specification. 2.10.2. Implicit Keep Previous experience with filtering systems suggests that cases tend to be missed in scripts. To prevent errors, Sieve has an "implicit keep". An implicit keep is a keep action (see 4.4) performed in absence of any action that cancels the implicit keep. An implicit keep is performed if a message is not written to a mailbox, redirected to a new address, or explicitly thrown out. That is, if a fileinto, a keep, a redirect, or a discard is performed, an implicit keep is not. Some actions may be defined to not cancel the implicit keep. These actions may not directly affect the delivery of a message, and are used for their side effects. None of the actions specified in this document meet that criteria, but extension actions will. For instance, with any of the short messages offered above, the following script produces no actions. Example: if size :over 500K { discard; } As a result, the implicit keep is taken. 2.10.3. Message Uniqueness in a Mailbox Implementations SHOULD NOT deliver a message to the same folder more than once, even if a script explicitly asks for a message to be written to a mailbox twice. Showalter Standards Track [Page 15] RFC 3028 Sieve: A Mail Filtering Language January 2001 The test for equality of two messages is implementation-defined. If a script asks for a message to be written to a mailbox twice, it MUST NOT be treated as an error. 2.10.4. Limits on Numbers of Actions Site policy MAY limit numbers of actions taken and MAY impose restrictions on which actions can be used together. In the event that a script hits a policy limit on the number of actions taken for a particular message, an error occurs. Implementations MUST prohibit more than one reject. Implementations MUST allow at least one keep or one fileinto. If fileinto is not implemented, implementations MUST allow at least one keep. Implementations SHOULD prohibit reject when used with other actions. 2.10.5. Extensions and Optional Features Because of the differing capabilities of many mail systems, several features of this specification are optional. Before any of these extensions can be executed, they must be declared with the "require" action. If an extension is not enabled with "require", implementations MUST treat it as if they did not support it at all. If a script does not understand an extension declared with require, the script must not be used at all. Implementations MUST NOT execute scripts which require unknown capability names. Note: The reason for this restriction is that prior experiences with languages such as LISP and Tcl suggest that this is a workable way of noting that a given script uses an extension. Experience with PostScript suggests that mechanisms that allow a script to work around missing extensions are not used in practice. Extensions which define actions MUST state how they interact with actions discussed in the base specification. Showalter Standards Track [Page 16] RFC 3028 Sieve: A Mail Filtering Language January 2001 2.10.6. Errors In any programming language, there are compile-time and run-time errors. Compile-time errors are ones in syntax that are detectable if a syntax check is done. Run-time errors are not detectable until the script is run. This includes transient failures like disk full conditions, but also includes issues like invalid combinations of actions. When an error occurs in a Sieve script, all processing stops. Implementations MAY choose to do a full parse, then evaluate the script, then do all actions. Implementations might even go so far as to ensure that execution is atomic (either all actions are executed or none are executed). Other implementations may choose to parse and run at the same time. Such implementations are simpler, but have issues with partial failure (some actions happen, others don't). Implementations might even go so far as to ensure that scripts can never execute an invalid set of actions (e.g., reject + fileinto) before execution, although this could involve solving the Halting Problem. This specification allows any of these approaches. Solving the Halting Problem is considered extra credit. When an error happens, implementations MUST notify the user that an error occurred, which actions (if any) were taken, and do an implicit keep. 2.10.7. Limits on Execution Implementations may limit certain constructs. However, this specification places a lower bound on some of these limits. Implementations MUST support fifteen levels of nested blocks. Implementations MUST support fifteen levels of nested test lists. 3. Control Commands Control structures are needed to allow for multiple and conditional actions. Showalter Standards Track [Page 17] RFC 3028 Sieve: A Mail Filtering Language January 2001 3.1. Control Structure If There are three pieces to if: "if", "elsif", and "else". Each is actually a separate command in terms of the grammar. However, an elsif MUST only follow an if, and an else MUST follow only either an if or an elsif. An error occurs if these conditions are not met. Syntax: if Syntax: elsif Syntax: else The semantics are similar to those of any of the many other programming languages these control commands appear in. When the interpreter sees an "if", it evaluates the test associated with it. If the test is true, it executes the block associated with it. If the test of the "if" is false, it evaluates the test of the first "elsif" (if any). If the test of "elsif" is true, it runs the elsif's block. An elsif may be followed by an elsif, in which case, the interpreter repeats this process until it runs out of elsifs. When the interpreter runs out of elsifs, there may be an "else" case. If there is, and none of the if or elsif tests were true, the interpreter runs the else case. This provides a way of performing exactly one of the blocks in the chain. In the following example, both Message A and B are dropped. Example: require "fileinto"; if header :contains "from" "coyote" { discard; } elsif header :contains ["subject"] ["$$$"] { discard; } else { fileinto "INBOX"; } When the script below is run over message A, it redirects the message to acm@example.edu; message B, to postmaster@example.edu; any other message is redirected to field@example.edu. Showalter Standards Track [Page 18] RFC 3028 Sieve: A Mail Filtering Language January 2001 Example: if header :contains ["From"] ["coyote"] { redirect "acm@example.edu"; } elsif header :contains "Subject" "$$$" { redirect "postmaster@example.edu"; } else { redirect "field@example.edu"; } Note that this definition prohibits the "... else if ..." sequence used by C. This is intentional, because this construct produces a shift-reduce conflict. 3.2. Control Structure Require Syntax: require The require action notes that a script makes use of a certain extension. Such a declaration is required to use the extension, as discussed in section 2.10.5. Multiple capabilities can be declared with a single require. The require command, if present, MUST be used before anything other than a require can be used. An error occurs if a require appears after a command other than require. Example: require ["fileinto", "reject"]; Example: require "fileinto"; require "vacation"; 3.3. Control Structure Stop Syntax: stop The "stop" action ends all processing. If no actions have been executed, then the keep action is taken. 4. Action Commands This document supplies five actions that may be taken on a message: keep, fileinto, redirect, reject, and discard. Implementations MUST support the "keep", "discard", and "redirect" actions. Implementations SHOULD support "reject" and "fileinto". Showalter Standards Track [Page 19] RFC 3028 Sieve: A Mail Filtering Language January 2001 Implementations MAY limit the number of certain actions taken (see section 2.10.4). 4.1. Action reject Syntax: reject The optional "reject" action refuses delivery of a message by sending back an [MDN] to the sender. It resends the message to the sender, wrapping it in a "reject" form, noting that it was rejected by the recipient. In the following script, message A is rejected and returned to the sender. Example: if header :contains "from" "coyote@desert.example.org" { reject "I am not taking mail from you, and I don't want your birdseed, either!"; } A reject message MUST take the form of a failure MDN as specified by [MDN]. The human-readable portion of the message, the first component of the MDN, contains the human readable message describing the error, and it SHOULD contain additional text alerting the original sender that mail was refused by a filter. This part of the MDN might appear as follows: ------------------------------------------------------------ Message was refused by recipient's mail filtering program. Reason given was as follows: I am not taking mail from you, and I don't want your birdseed, either! ------------------------------------------------------------ The MDN action-value field as defined in the MDN specification MUST be "deleted" and MUST have the MDN-sent-automatically and automatic- action modes set. Because some implementations can not or will not implement the reject command, it is optional. The capability string to be used with the require command is "reject". 4.2. Action fileinto Syntax: fileinto The "fileinto" action delivers the message into the specified folder. Implementations SHOULD support fileinto, but in some environments this may be impossible. Showalter Standards Track [Page 20] RFC 3028 Sieve: A Mail Filtering Language January 2001 The capability string for use with the require command is "fileinto". In the following script, message A is filed into folder "INBOX.harassment". Example: require "fileinto"; if header :contains ["from"] "coyote" { fileinto "INBOX.harassment"; } 4.3. Action redirect Syntax: redirect The "redirect" action is used to send the message to another user at a supplied address, as a mail forwarding feature does. The "redirect" action makes no changes to the message body or existing headers, but it may add new headers. The "redirect" modifies the envelope recipient. The redirect command performs an MTA-style "forward"--that is, what you get from a .forward file using sendmail under UNIX. The address on the SMTP envelope is replaced with the one on the redirect command and the message is sent back out. (This is not an MUA-style forward, which creates a new message with a different sender and message ID, wrapping the old message in a new one.) A simple script can be used for redirecting all mail: Example: redirect "bart@example.edu"; Implementations SHOULD take measures to implement loop control, possibly including adding headers to the message or counting received headers. If an implementation detects a loop, it causes an error. 4.4. Action keep Syntax: keep The "keep" action is whatever action is taken in lieu of all other actions, if no filtering happens at all; generally, this simply means to file the message into the user's main mailbox. This command provides a way to execute this action without needing to know the name of the user's main mailbox, providing a way to call it without needing to understand the user's setup, or the underlying mail system. Showalter Standards Track [Page 21] RFC 3028 Sieve: A Mail Filtering Language January 2001 For instance, in an implementation where the IMAP server is running scripts on behalf of the user at time of delivery, a keep command is equivalent to a fileinto "INBOX". Example: if size :under 1M { keep; } else { discard; } Note that the above script is identical to the one below. Example: if not size :under 1M { discard; } 4.5. Action discard Syntax: discard Discard is used to silently throw away the message. It does so by simply canceling the implicit keep. If discard is used with other actions, the other actions still happen. Discard is compatible with all other actions. (For instance fileinto+discard is equivalent to fileinto.) Discard MUST be silent; that is, it MUST NOT return a non-delivery notification of any kind ([DSN], [MDN], or otherwise). In the following script, any mail from "idiot@example.edu" is thrown out. Example: if header :contains ["from"] ["idiot@example.edu"] { discard; } While an important part of this language, "discard" has the potential to create serious problems for users: Students who leave themselves logged in to an unattended machine in a public computer lab may find their script changed to just "discard". In order to protect users in this situation (along with similar situations), implementations MAY keep messages destroyed by a script for an indefinite period, and MAY disallow scripts that throw out all mail. 5. Test Commands Tests are used in conditionals to decide which part(s) of the conditional to execute. Implementations MUST support these tests: "address", "allof", "anyof", "exists", "false", "header", "not", "size", and "true". Implementations SHOULD support the "envelope" test. Showalter Standards Track [Page 22] RFC 3028 Sieve: A Mail Filtering Language January 2001 5.1. Test address Syntax: address [ADDRESS-PART] [COMPARATOR] [MATCH-TYPE] The address test matches Internet addresses in structured headers that contain addresses. It returns true if any header contains any key in the specified part of the address, as modified by the comparator and the match keyword. Like envelope and header, this test returns true if any combination of the header-list and key-list arguments match. Internet email addresses [IMAIL] have the somewhat awkward characteristic that the local-part to the left of the at-sign is considered case sensitive, and the domain-part to the right of the at-sign is case insensitive. The "address" command does not deal with this itself, but provides the ADDRESS-PART argument for allowing users to deal with it. The address primitive never acts on the phrase part of an email address, nor on comments within that address. It also never acts on group names, although it does act on the addresses within the group construct. Implementations MUST restrict the address test to headers that contain addresses, but MUST include at least From, To, Cc, Bcc, Sender, Resent-From, Resent-To, and SHOULD include any other header that utilizes an "address-list" structured header body. Example: if address :is :all "from" "tim@example.com" { discard; 5.2. Test allof Syntax: allof The allof test performs a logical AND on the tests supplied to it. Example: allof (false, false) => false allof (false, true) => false allof (true, true) => true The allof test takes as its argument a test-list. Showalter Standards Track [Page 23] RFC 3028 Sieve: A Mail Filtering Language January 2001 5.3. Test anyof Syntax: anyof The anyof test performs a logical OR on the tests supplied to it. Example: anyof (false, false) => false anyof (false, true) => true anyof (true, true) => true 5.4. Test envelope Syntax: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] The "envelope" test is true if the specified part of the SMTP (or equivalent) envelope matches the specified key. If one of the envelope-part strings is (case insensitive) "from", then matching occurs against the FROM address used in the SMTP MAIL command. If one of the envelope-part strings is (case insensitive) "to", then matching occurs against the TO address used in the SMTP RCPT command that resulted in this message getting delivered to this user. Note that only the most recent TO is available, and only the one relevant to this user. The envelope-part is a string list and may contain more than one parameter, in which case all of the strings specified in the key-list are matched against all parts given in the envelope-part list. Like address and header, this test returns true if any combination of the envelope-part and key-list arguments is true. All tests against envelopes MUST drop source routes. If the SMTP transaction involved several RCPT commands, only the data from the RCPT command that caused delivery to this user is available in the "to" part of the envelope. If a protocol other than SMTP is used for message transport, implementations are expected to adapt this command appropriately. The envelope command is optional. Implementations SHOULD support it, but the necessary information may not be available in all cases. Showalter Standards Track [Page 24] RFC 3028 Sieve: A Mail Filtering Language January 2001 Example: require "envelope"; if envelope :all :is "from" "tim@example.com" { discard; } 5.5. Test exists Syntax: exists The "exists" test is true if the headers listed in the header-names argument exist within the message. All of the headers must exist or the test is false. The following example throws out mail that doesn't have a From header and a Date header. Example: if not exists ["From","Date"] { discard; } 5.6. Test false Syntax: false The "false" test always evaluates to false. 5.7. Test header Syntax: header [COMPARATOR] [MATCH-TYPE] The "header" test evaluates to true if any header name matches any key. The type of match is specified by the optional match argument, which defaults to ":is" if not specified, as specified in section 2.6. Like address and envelope, this test returns true if any combination of the string-list and key-list arguments match. If a header listed in the header-names argument exists, it contains the null key (""). However, if the named header is not present, it does not contain the null key. So if a message contained the header X-Caffeine: C8H10N4O2 Showalter Standards Track [Page 25] RFC 3028 Sieve: A Mail Filtering Language January 2001 these tests on that header evaluate as follows: header :is ["X-Caffeine"] [""] => false header :contains ["X-Caffeine"] [""] => true 5.8. Test not Syntax: not The "not" test takes some other test as an argument, and yields the opposite result. "not false" evaluates to "true" and "not true" evaluates to "false". 5.9. Test size Syntax: size <":over" / ":under"> The "size" test deals with the size of a message. It takes either a tagged argument of ":over" or ":under", followed by a number representing the size of the message. If the argument is ":over", and the size of the message is greater than the number provided, the test is true; otherwise, it is false. If the argument is ":under", and the size of the message is less than the number provided, the test is true; otherwise, it is false. Exactly one of ":over" or ":under" must be specified, and anything else is an error. The size of a message is defined to be the number of octets from the initial header until the last character in the message body. Note that for a message that is exactly 4,000 octets, the message is neither ":over" 4000 octets or ":under" 4000 octets. 5.10. Test true Syntax: true The "true" test always evaluates to true. 6. Extensibility New control structures, actions, and tests can be added to the language. Sites must make these features known to their users; this document does not define a way to discover the list of extensions supported by the server. Showalter Standards Track [Page 26] RFC 3028 Sieve: A Mail Filtering Language January 2001 Any extensions to this language MUST define a capability string that uniquely identifies that extension. If a new version of an extension changes the functionality of a previously defined extension, it MUST use a different name. In a situation where there is a submission protocol and an extension advertisement mechanism aware of the details of this language, scripts submitted can be checked against the mail server to prevent use of an extension that the server does not support. Extensions MUST state how they interact with constraints defined in section 2.10, e.g., whether they cancel the implicit keep, and which actions they are compatible and incompatible with. 6.1. Capability String Capability strings are typically short strings describing what capabilities are supported by the server. Capability strings beginning with "vnd." represent vendor-defined extensions. Such extensions are not defined by Internet standards or RFCs, but are still registered with IANA in order to prevent conflicts. Extensions starting with "vnd." SHOULD be followed by the name of the vendor and product, such as "vnd.acme.rocket-sled". The following capability strings are defined by this document: envelope The string "envelope" indicates that the implementation supports the "envelope" command. fileinto The string "fileinto" indicates that the implementation supports the "fileinto" command. reject The string "reject" indicates that the implementation supports the "reject" command. comparator- The string "comparator-elbonia" is provided if the implementation supports the "elbonia" comparator. Therefore, all implementations have at least the "comparator-i;octet" and "comparator-i;ascii-casemap" capabilities. However, these comparators may be used without being declared with require. Showalter Standards Track [Page 27] RFC 3028 Sieve: A Mail Filtering Language January 2001 6.2. IANA Considerations In order to provide a standard set of extensions, a registry is provided by IANA. Capability names may be registered on a first- come, first-served basis. Extensions designed for interoperable use SHOULD be defined as standards track or IESG approved experimental RFCs. 6.2.1. Template for Capability Registrations The following template is to be used for registering new Sieve extensions with IANA. To: iana@iana.org Subject: Registration of new Sieve extension Capability name: Capability keyword: Capability arguments: Standards Track/IESG-approved experimental RFC number: Person and email address to contact for further information: 6.2.2. Initial Capability Registrations The following are to be added to the IANA registry for Sieve extensions as the initial contents of the capability registry. Capability name: fileinto Capability keyword: fileinto Capability arguments: fileinto Standards Track/IESG-approved experimental RFC number: RFC 3028 (Sieve base spec) Person and email address to contact for further information: Tim Showalter tjs@mirapoint.com Capability name: reject Capability keyword: reject Capability arguments: reject Standards Track/IESG-approved experimental RFC number: RFC 3028 (Sieve base spec) Person and email address to contact for further information: Tim Showalter tjs@mirapoint.com Showalter Standards Track [Page 28] RFC 3028 Sieve: A Mail Filtering Language January 2001 Capability name: envelope Capability keyword: envelope Capability arguments: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] Standards Track/IESG-approved experimental RFC number: RFC 3028 (Sieve base spec) Person and email address to contact for further information: Tim Showalter tjs@mirapoint.com Capability name: comparator-* Capability keyword: comparator-* (anything starting with "comparator-") Capability arguments: (none) Standards Track/IESG-approved experimental RFC number: RFC 3028, Sieve, by reference of RFC 2244, Application Configuration Access Protocol Person and email address to contact for further information: Tim Showalter tjs@mirapoint.com 6.3. Capability Transport As the range of mail systems that this document is intended to apply to is quite varied, a method of advertising which capabilities an implementation supports is difficult due to the wide range of possible implementations. Such a mechanism, however, should have property that the implementation can advertise the complete set of extensions that it supports. 7. Transmission The MIME type for a Sieve script is "application/sieve". The registration of this type for RFC 2048 requirements is as follows: Subject: Registration of MIME media type application/sieve MIME media type name: application MIME subtype name: sieve Required parameters: none Optional parameters: none Encoding considerations: Most sieve scripts will be textual, written in UTF-8. When non-7bit characters are used, quoted-printable is appropriate for transport systems that require 7bit encoding. Showalter Standards Track [Page 29] RFC 3028 Sieve: A Mail Filtering Language January 2001 Security considerations: Discussed in section 10 of RFC 3028. Interoperability considerations: Discussed in section 2.10.5 of RFC 3028. Published specification: RFC 3028. Applications which use this media type: sieve-enabled mail servers Additional information: Magic number(s): File extension(s): .siv Macintosh File Type Code(s): Person & email address to contact for further information: See the discussion list at ietf-mta-filters@imc.org. Intended usage: COMMON Author/Change controller: See Author information in RFC 3028. 8. Parsing The Sieve grammar is separated into tokens and a separate grammar as most programming languages are. 8.1. Lexical Tokens Sieve scripts are encoded in UTF-8. The following assumes a valid UTF-8 encoding; special characters in Sieve scripts are all ASCII. The following are tokens in Sieve: - identifiers - tags - numbers - quoted strings - multi-line strings - other separators Blanks, horizontal tabs, CRLFs, and comments ("white space") are ignored except as they separate tokens. Some white space is required to separate otherwise adjacent tokens and in specific places in the multi-line strings. The other separators are single individual characters, and are mentioned explicitly in the grammar. The lexical structure of sieve is defined in the following BNF (as described in [ABNF]): Showalter Standards Track [Page 30] RFC 3028 Sieve: A Mail Filtering Language January 2001 bracket-comment = "/*" *(CHAR-NOT-STAR / ("*" CHAR-NOT-SLASH)) "*/" ;; No */ allowed inside a comment. ;; (No * is allowed unless it is the last character, ;; or unless it is followed by a character that isn't a ;; slash.) CHAR-NOT-DOT = (%x01-09 / %x0b-0c / %x0e-2d / %x2f-ff) ;; no dots, no CRLFs CHAR-NOT-CRLF = (%x01-09 / %x0b-0c / %x0e-ff) CHAR-NOT-SLASH = (%x00-57 / %x58-ff) CHAR-NOT-STAR = (%x00-51 / %x53-ff) comment = bracket-comment / hash-comment hash-comment = ( "#" *CHAR-NOT-CRLF CRLF ) identifier = (ALPHA / "_") *(ALPHA DIGIT "_") tag = ":" identifier number = 1*DIGIT [QUANTIFIER] QUANTIFIER = "K" / "M" / "G" quoted-string = DQUOTE *CHAR DQUOTE ;; in general, \ CHAR inside a string maps to CHAR ;; so \" maps to " and \\ maps to \ ;; note that newlines and other characters are all allowed ;; strings multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) *(multi-line-literal / multi-line-dotstuff) "." CRLF multi-line-literal = [CHAR-NOT-DOT *CHAR-NOT-CRLF] CRLF multi-line-dotstuff = "." 1*CHAR-NOT-CRLF CRLF ;; A line containing only "." ends the multi-line. ;; Remove a leading '.' if followed by another '.'. white-space = 1*(SP / CRLF / HTAB) / comment 8.2. Grammar The following is the grammar of Sieve after it has been lexically interpreted. No white space or comments appear below. The start symbol is "start". Showalter Standards Track [Page 31] RFC 3028 Sieve: A Mail Filtering Language January 2001 argument = string-list / number / tag arguments = *argument [test / test-list] block = "{" commands "}" command = identifier arguments ( ";" / block ) commands = *command start = commands string = quoted-string / multi-line string-list = "[" string *("," string) "]" / string ;; if there is only a single string, the brackets are optional test = identifier arguments test-list = "(" test *("," test) ")" 9. Extended Example The following is an extended example of a Sieve script. Note that it does not make use of the implicit keep. # # Example Sieve Filter # Declare any optional features or extension used by the script # require ["fileinto", "reject"]; # # Reject any large messages (note that the four leading dots get # "stuffed" to three) # if size :over 1M { reject text: Please do not send me large attachments. Put your file on a server and send me the URL. Thank you. .... Fred . ; stop; } # Showalter Standards Track [Page 32] RFC 3028 Sieve: A Mail Filtering Language January 2001 # Handle messages from known mailing lists # Move messages from IETF filter discussion list to filter folder # if header :is "Sender" "owner-ietf-mta-filters@imc.org" { fileinto "filter"; # move to "filter" folder } # # Keep all messages to or from people in my company # elsif address :domain :is ["From", "To"] "example.com" { keep; # keep in "In" folder } # # Try and catch unsolicited email. If a message is not to me, # or it contains a subject known to be spam, file it away. # elsif anyof (not address :all :contains ["To", "Cc", "Bcc"] "me@example.com", header :matches "subject" ["*make*money*fast*", "*university*dipl*mas*"]) { # If message header does not contain my address, # it's from a list. fileinto "spam"; # move to "spam" folder } else { # Move all other (non-company) mail to "personal" # folder. fileinto "personal"; } Showalter Standards Track [Page 33] RFC 3028 Sieve: A Mail Filtering Language January 2001 10. Security Considerations Users must get their mail. It is imperative that whatever method implementations use to store the user-defined filtering scripts be secure. It is equally important that implementations sanity-check the user's scripts, and not allow users to create on-demand mailbombs. For instance, an implementation that allows a user to reject or redirect multiple times to a single message might also allow a user to create a mailbomb triggered by mail from a specific user. Site- or implementation-defined limits on actions are useful for this. Several commands, such as "discard", "redirect", and "fileinto" allow for actions to be taken that are potentially very dangerous. Implementations SHOULD take measures to prevent languages from looping. 11. Acknowledgments I am very thankful to Chris Newman for his support and his ABNF syntax checker, to John Myers and Steve Hole for outlining the requirements for the original drafts, to Larry Greenfield for nagging me about the grammar and finally fixing it, to Greg Sereda for repeatedly fixing and providing examples, to Ned Freed for fixing everything else, to Rob Earhart for an early implementation and a great deal of help, and to Randall Gellens for endless amounts of proofreading. I am grateful to Carnegie Mellon University where most of the work on this document was done. I am also indebted to all of the readers of the ietf-mta-filters@imc.org mailing list. 12. Author's Address Tim Showalter Mirapoint, Inc. 909 Hermosa Court Sunnyvale, CA 94085 EMail: tjs@mirapoint.com 13. References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. Showalter Standards Track [Page 34] RFC 3028 Sieve: A Mail Filtering Language January 2001 [ACAP] Newman, C. and J. G. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in electrical technology - Part 2: Telecommunications and electronics", January 1999. [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format for Delivery Status Notifications", RFC 1894, January 1996. [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and Cooperative Work in a Practical Multimedia Message System", Int. J. of Man-Machine Studies, April, 1991. Reprinted in Computer-Supported Cooperative Work and Groupware, Saul Greenberg, editor, Harcourt Brace Jovanovich, 1991. Reprinted in Readings in Groupware and Computer-Supported Cooperative Work, Ronald Baecker, editor, Morgan Kaufmann, 1993. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [IMAP] Crispin, M., "Internet Message Access Protocol - version 4rev1", RFC 2060, December 1996. [IMAIL] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822, August 1982. [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [MDN] Fajman, R., "An Extensible Message Format for Message Disposition Notifications", RFC 2298, March 1998. [RFC1123] Braden, R., "Requirements for Internet Hosts -- Application and Support", STD 3, RFC 1123, November 1989. [SMTP] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC 821, August 1982. [UTF-8] Yergeau, F., "UTF-8, a transformation format of Unicode and ISO 10646", RFC 2044, October 1996. Showalter Standards Track [Page 35] RFC 3028 Sieve: A Mail Filtering Language January 2001 14. Full Copyright Statement Copyright (C) The Internet Society (2001). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Showalter Standards Track [Page 36] ================================================ FILE: docs/rfcs/rfc3348.IMAP4_Child_Mailbox_extension.txt ================================================ Network Working Group M. Gahrns Request for Comments: 3348 R. Cheng Category: Informational Microsoft July 2002 The Internet Message Action Protocol (IMAP4) Child Mailbox Extension Status of this Memo This memo provides information for the Internet community. It does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2002). All Rights Reserved. Abstract The Internet Message Action Protocol (IMAP4) CHILDREN extension provides a mechanism for a client to efficiently determine if a particular mailbox has children, without issuing a LIST "" * or a LIST "" % for each mailbox. 1. Conventions used in this document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. If such lines are wrapped without a new "C:" or "S:" label, then the wrapping is for editorial clarity and is not part of the command. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC-2119]. 2. Introduction and Overview Many IMAP4 [RFC-2060] clients present to the user a hierarchical view of the mailboxes that a user has access to. Rather than initially presenting to the user the entire mailbox hierarchy, it is often preferable to show to the user a collapsed outline list of the mailbox hierarchy (particularly if there is a large number of mailboxes). The user can then expand the collapsed outline hierarchy as needed. It is common to include within the collapsed hierarchy a Gahrns, et al. Informational [Page 1] RFC 3348 IMAP4 Child Mailbox Extension July 2002 visual clue (such as a "+") to indicate that there are child mailboxes under a particular mailbox. When the visual clue is clicked the hierarchy list is expanded to show the child mailboxes. Several IMAP vendors implemented this proposal, and it is proposed to document this behavior and functionality as an Informational RFC. There is interest in addressing the general extensibility of the IMAP LIST command through an IMAP LIST Extension draft. Similar functionality to the \HasChildren and \HasNoChildren flags could be incorporated into this new LIST Extension. It is proposed that the more general LIST Extension draft proceed on the standards track with this proposal being relegated to informational status only. If the functionality of the \HasChildren and \HasNoChildren flags were incorporated into a more general LIST extension, this would have the advantage that a client could then have the opportunity to request whether or not the server should return this information. This would be an advantage over the current draft for servers where this information is expensive to compute, since the server would only need to compute the information when it knew that the client requesting the information was able to consume it. 3. Requirements IMAP4 servers that support this extension MUST list the keyword CHILDREN in their CAPABILITY response. The CHILDREN extension defines two new attributes that MAY be returned within a LIST response. \HasChildren - The presence of this attribute indicates that the mailbox has child mailboxes. Servers SHOULD NOT return \HasChildren if child mailboxes exist, but none will be displayed to the current user in a LIST response (as should be the case where child mailboxes exist, but a client does not have permissions to access them.) In this case, \HasNoChildren SHOULD be used. In many cases, however, a server may not be able to efficiently compute whether a user has access to all child mailboxes, or multiple users may be accessing the same account and simultaneously changing the mailbox hierarchy. As such a client MUST be prepared to accept the \HasChildren attribute as a hint. That is, a mailbox MAY be flagged with the \HasChildren attribute, but no child mailboxes will appear in a subsequent LIST response. Gahrns, et al. Informational [Page 2] RFC 3348 IMAP4 Child Mailbox Extension July 2002 Example 3.1: ============ /*** Consider a server that has the following mailbox hierarchy: INBOX ITEM_1 ITEM_1A ITEM_2 TOP_SECRET Where INBOX, ITEM_1 and ITEM_2 are top level mailboxes. ITEM_1A is a child mailbox of ITEM_1 and TOP_SECRET is a child mailbox of ITEM_2 that the currently logged on user does NOT have access to. Note that in this case, the server is not able to efficiently compute access rights to child mailboxes and responds with a \HasChildren attribute for mailbox ITEM_2, even though ITEM_2/TOP_SECRET does not appear in the list response. ***/ C: A001 LIST "" * S: * LIST (\HasNoChildren) "/" INBOX S: * LIST (\HasChildren) "/" ITEM_1 S: * LIST (\HasNoChildren) "/" ITEM_1/ITEM_1A S: * LIST (\HasChildren) "/" ITEM_2 S: A001 OK LIST Completed \HasNoChildren - The presence of this attribute indicates that the mailbox has NO child mailboxes that are accessible to the currently authenticated user. If a mailbox has the \Noinferiors attribute, the \HasNoChildren attribute is redundant and SHOULD be omitted in the LIST response. In some instances a server that supports the CHILDREN extension MAY NOT be able to determine whether a mailbox has children. For example it may have difficulty determining whether there are child mailboxes when LISTing mailboxes while operating in a particular namespace. In these cases, a server MAY exclude both the \HasChildren and \HasNoChildren attributes in the LIST response. As such, a client can not make any assumptions about whether a mailbox has children based upon the absence of a single attribute. It is an error for the server to return both a \HasChildren and a \HasNoChildren attribute in a LIST response. Gahrns, et al. Informational [Page 3] RFC 3348 IMAP4 Child Mailbox Extension July 2002 It is an error for the server to return both a \HasChildren and a \NoInferiors attribute in a LIST response. Note: the \HasNoChildren attribute should not be confused with the IMAP4 [RFC-2060] defined attribute \Noinferiors which indicates that no child mailboxes exist now and none can be created in the future. The \HasChildren and \HasNoChildren attributes might not be returned in response to a LSUB response. Many servers maintain a simple mailbox subscription list that is not updated when the underlying mailbox structure is changed. A client MUST NOT assume that hierarchy information will be maintained in the subscription list. RLIST is a command defined in [RFC-2193] that includes in a LIST response mailboxes that are accessible only via referral. That is, a client must explicitly issue an RLIST command to see a list of these mailboxes. Thus in the case where a mailbox has child mailboxes that are available only via referral, the mailboxes would appear as \HasNoChildren in response to the LIST command, and \HasChildren in response to the RLIST command. 5. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. Two new mailbox attributes are defined as flag_extensions to the IMAP4 mailbox_list response: HasChildren = "\HasChildren" HasNoChildren = "\HasNoChildren" 6. Security Considerations This extension provides a client a more efficient means of determining whether a particular mailbox has children. If a mailbox has children, but the currently authenticated user does not have access to any of them, the server SHOULD respond with a \HasNoChildren attribute. In many cases, however, a server may not be able to efficiently compute whether a user has access to all child mailboxes. If such a server responds with a \HasChildren attribute, when in fact the currently authenticated user does not have access to any child mailboxes, potentially more information is conveyed about the mailbox than intended. A server designed with such levels of security in mind SHOULD NOT attach the \HasChildren attribute to a mailbox unless the server is certain that the user has access to at least one of the child mailboxes. Gahrns, et al. Informational [Page 4] RFC 3348 IMAP4 Child Mailbox Extension July 2002 7. References [RFC-2060] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 2060, December 1996. [RFC-2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC-2234] Crocker, D. and P. Overell, Editors, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [RFC-2193] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, September 1997. 8. Acknowledgments The authors would like to thank the participants of several IMC Mail Connect events for their input when this idea was originally presented and refined. 9. Author's Address Mike Gahrns Microsoft One Microsoft Way Redmond, WA, 98052 Phone: (425) 936-9833 EMail: mikega@microsoft.com Raymond Cheng Microsoft One Microsoft Way Redmond, WA, 98052 Phone: (425) 703-4913 EMail: raych@microsoft.com Gahrns, et al. Informational [Page 5] RFC 3348 IMAP4 Child Mailbox Extension July 2002 10. Full Copyright Statement Copyright (C) The Internet Society (2002). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Gahrns, et al. Informational [Page 6] ================================================ FILE: docs/rfcs/rfc3501.IMAP4rev1.txt ================================================ Network Working Group M. Crispin Request for Comments: 3501 University of Washington Obsoletes: 2060 March 2003 Category: Standards Track INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1 Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2003). All Rights Reserved. Abstract The Internet Message Access Protocol, Version 4rev1 (IMAP4rev1) allows a client to access and manipulate electronic mail messages on a server. IMAP4rev1 permits manipulation of mailboxes (remote message folders) in a way that is functionally equivalent to local folders. IMAP4rev1 also provides the capability for an offline client to resynchronize with the server. IMAP4rev1 includes operations for creating, deleting, and renaming mailboxes, checking for new messages, permanently removing messages, setting and clearing flags, RFC 2822 and RFC 2045 parsing, searching, and selective fetching of message attributes, texts, and portions thereof. Messages in IMAP4rev1 are accessed by the use of numbers. These numbers are either message sequence numbers or unique identifiers. IMAP4rev1 supports a single server. A mechanism for accessing configuration information to support multiple IMAP4rev1 servers is discussed in RFC 2244. IMAP4rev1 does not specify a means of posting mail; this function is handled by a mail transfer protocol such as RFC 2821. Crispin Standards Track [Page 1] RFC 3501 IMAPv4 March 2003 Table of Contents IMAP4rev1 Protocol Specification ................................ 4 1. How to Read This Document ............................... 4 1.1. Organization of This Document ........................... 4 1.2. Conventions Used in This Document ....................... 4 1.3. Special Notes to Implementors ........................... 5 2. Protocol Overview ....................................... 6 2.1. Link Level .............................................. 6 2.2. Commands and Responses .................................. 6 2.2.1. Client Protocol Sender and Server Protocol Receiver ..... 6 2.2.2. Server Protocol Sender and Client Protocol Receiver ..... 7 2.3. Message Attributes ...................................... 8 2.3.1. Message Numbers ......................................... 8 2.3.1.1. Unique Identifier (UID) Message Attribute ....... 8 2.3.1.2. Message Sequence Number Message Attribute ....... 10 2.3.2. Flags Message Attribute ................................. 11 2.3.3. Internal Date Message Attribute ......................... 12 2.3.4. [RFC-2822] Size Message Attribute ....................... 12 2.3.5. Envelope Structure Message Attribute .................... 12 2.3.6. Body Structure Message Attribute ........................ 12 2.4. Message Texts ........................................... 13 3. State and Flow Diagram .................................. 13 3.1. Not Authenticated State ................................. 13 3.2. Authenticated State ..................................... 13 3.3. Selected State .......................................... 13 3.4. Logout State ............................................ 14 4. Data Formats ............................................ 16 4.1. Atom .................................................... 16 4.2. Number .................................................. 16 4.3. String .................................................. 16 4.3.1. 8-bit and Binary Strings ................................ 17 4.4. Parenthesized List ...................................... 17 4.5. NIL ..................................................... 17 5. Operational Considerations .............................. 18 5.1. Mailbox Naming .......................................... 18 5.1.1. Mailbox Hierarchy Naming ................................ 19 5.1.2. Mailbox Namespace Naming Convention ..................... 19 5.1.3. Mailbox International Naming Convention ................. 19 5.2. Mailbox Size and Message Status Updates ................. 21 5.3. Response when no Command in Progress .................... 21 5.4. Autologout Timer ........................................ 22 5.5. Multiple Commands in Progress ........................... 22 6. Client Commands ........................................ 23 6.1. Client Commands - Any State ............................ 24 6.1.1. CAPABILITY Command ..................................... 24 6.1.2. NOOP Command ........................................... 25 6.1.3. LOGOUT Command ......................................... 26 Crispin Standards Track [Page 2] RFC 3501 IMAPv4 March 2003 6.2. Client Commands - Not Authenticated State .............. 26 6.2.1. STARTTLS Command ....................................... 27 6.2.2. AUTHENTICATE Command ................................... 28 6.2.3. LOGIN Command .......................................... 30 6.3. Client Commands - Authenticated State .................. 31 6.3.1. SELECT Command ......................................... 32 6.3.2. EXAMINE Command ........................................ 34 6.3.3. CREATE Command ......................................... 34 6.3.4. DELETE Command ......................................... 35 6.3.5. RENAME Command ......................................... 37 6.3.6. SUBSCRIBE Command ...................................... 39 6.3.7. UNSUBSCRIBE Command .................................... 39 6.3.8. LIST Command ........................................... 40 6.3.9. LSUB Command ........................................... 43 6.3.10. STATUS Command ......................................... 44 6.3.11. APPEND Command ......................................... 46 6.4. Client Commands - Selected State ....................... 47 6.4.1. CHECK Command .......................................... 47 6.4.2. CLOSE Command .......................................... 48 6.4.3. EXPUNGE Command ........................................ 49 6.4.4. SEARCH Command ......................................... 49 6.4.5. FETCH Command .......................................... 54 6.4.6. STORE Command .......................................... 58 6.4.7. COPY Command ........................................... 59 6.4.8. UID Command ............................................ 60 6.5. Client Commands - Experimental/Expansion ............... 62 6.5.1. X Command ........................................ 62 7. Server Responses ....................................... 62 7.1. Server Responses - Status Responses .................... 63 7.1.1. OK Response ............................................ 65 7.1.2. NO Response ............................................ 66 7.1.3. BAD Response ........................................... 66 7.1.4. PREAUTH Response ....................................... 67 7.1.5. BYE Response ........................................... 67 7.2. Server Responses - Server and Mailbox Status ........... 68 7.2.1. CAPABILITY Response .................................... 68 7.2.2. LIST Response .......................................... 69 7.2.3. LSUB Response .......................................... 70 7.2.4 STATUS Response ........................................ 70 7.2.5. SEARCH Response ........................................ 71 7.2.6. FLAGS Response ......................................... 71 7.3. Server Responses - Mailbox Size ........................ 71 7.3.1. EXISTS Response ........................................ 71 7.3.2. RECENT Response ........................................ 72 7.4. Server Responses - Message Status ...................... 72 7.4.1. EXPUNGE Response ....................................... 72 7.4.2. FETCH Response ......................................... 73 7.5. Server Responses - Command Continuation Request ........ 79 Crispin Standards Track [Page 3] RFC 3501 IMAPv4 March 2003 8. Sample IMAP4rev1 connection ............................ 80 9. Formal Syntax .......................................... 81 10. Author's Note .......................................... 92 11. Security Considerations ................................ 92 11.1. STARTTLS Security Considerations ....................... 92 11.2. Other Security Considerations .......................... 93 12. IANA Considerations .................................... 94 Appendices ..................................................... 95 A. References ............................................. 95 B. Changes from RFC 2060 .................................. 97 C. Key Word Index ......................................... 103 Author's Address ............................................... 107 Full Copyright Statement ....................................... 108 IMAP4rev1 Protocol Specification 1. How to Read This Document 1.1. Organization of This Document This document is written from the point of view of the implementor of an IMAP4rev1 client or server. Beyond the protocol overview in section 2, it is not optimized for someone trying to understand the operation of the protocol. The material in sections 3 through 5 provides the general context and definitions with which IMAP4rev1 operates. Sections 6, 7, and 9 describe the IMAP commands, responses, and syntax, respectively. The relationships among these are such that it is almost impossible to understand any of them separately. In particular, do not attempt to deduce command syntax from the command section alone; instead refer to the Formal Syntax section. 1.2. Conventions Used in This Document "Conventions" are basic principles or procedures. Document conventions are noted in this section. In examples, "C:" and "S:" indicate lines sent by the client and server respectively. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. The word "can" (not "may") is used to refer to a possible circumstance or situation, as opposed to an optional facility of the protocol. Crispin Standards Track [Page 4] RFC 3501 IMAPv4 March 2003 "User" is used to refer to a human user, whereas "client" refers to the software being run by the user. "Connection" refers to the entire sequence of client/server interaction from the initial establishment of the network connection until its termination. "Session" refers to the sequence of client/server interaction from the time that a mailbox is selected (SELECT or EXAMINE command) until the time that selection ends (SELECT or EXAMINE of another mailbox, CLOSE command, or connection termination). Characters are 7-bit US-ASCII unless otherwise specified. Other character sets are indicated using a "CHARSET", as described in [MIME-IMT] and defined in [CHARSET]. CHARSETs have important additional semantics in addition to defining character set; refer to these documents for more detail. There are several protocol conventions in IMAP. These refer to aspects of the specification which are not strictly part of the IMAP protocol, but reflect generally-accepted practice. Implementations need to be aware of these conventions, and avoid conflicts whether or not they implement the convention. For example, "&" may not be used as a hierarchy delimiter since it conflicts with the Mailbox International Naming Convention, and other uses of "&" in mailbox names are impacted as well. 1.3. Special Notes to Implementors Implementors of the IMAP protocol are strongly encouraged to read the IMAP implementation recommendations document [IMAP-IMPLEMENTATION] in conjunction with this document, to help understand the intricacies of this protocol and how best to build an interoperable product. IMAP4rev1 is designed to be upwards compatible from the [IMAP2] and unpublished IMAP2bis protocols. IMAP4rev1 is largely compatible with the IMAP4 protocol described in RFC 1730; the exception being in certain facilities added in RFC 1730 that proved problematic and were subsequently removed. In the course of the evolution of IMAP4rev1, some aspects in the earlier protocols have become obsolete. Obsolete commands, responses, and data formats which an IMAP4rev1 implementation can encounter when used with an earlier implementation are described in [IMAP-OBSOLETE]. Other compatibility issues with IMAP2bis, the most common variant of the earlier protocol, are discussed in [IMAP-COMPAT]. A full discussion of compatibility issues with rare (and presumed extinct) Crispin Standards Track [Page 5] RFC 3501 IMAPv4 March 2003 variants of [IMAP2] is in [IMAP-HISTORICAL]; this document is primarily of historical interest. IMAP was originally developed for the older [RFC-822] standard, and as a consequence several fetch items in IMAP incorporate "RFC822" in their name. With the exception of RFC822.SIZE, there are more modern replacements; for example, the modern version of RFC822.HEADER is BODY.PEEK[HEADER]. In all cases, "RFC822" should be interpreted as a reference to the updated [RFC-2822] standard. 2. Protocol Overview 2.1. Link Level The IMAP4rev1 protocol assumes a reliable data stream such as that provided by TCP. When TCP is used, an IMAP4rev1 server listens on port 143. 2.2. Commands and Responses An IMAP4rev1 connection consists of the establishment of a client/server network connection, an initial greeting from the server, and client/server interactions. These client/server interactions consist of a client command, server data, and a server completion result response. All interactions transmitted by client and server are in the form of lines, that is, strings that end with a CRLF. The protocol receiver of an IMAP4rev1 client or server is either reading a line, or is reading a sequence of octets with a known count followed by a line. 2.2.1. Client Protocol Sender and Server Protocol Receiver The client command begins an operation. Each client command is prefixed with an identifier (typically a short alphanumeric string, e.g., A0001, A0002, etc.) called a "tag". A different tag is generated by the client for each command. Clients MUST follow the syntax outlined in this specification strictly. It is a syntax error to send a command with missing or extraneous spaces or arguments. There are two cases in which a line from the client does not represent a complete command. In one case, a command argument is quoted with an octet count (see the description of literal in String under Data Formats); in the other case, the command arguments require server feedback (see the AUTHENTICATE command). In either case, the Crispin Standards Track [Page 6] RFC 3501 IMAPv4 March 2003 server sends a command continuation request response if it is ready for the octets (if appropriate) and the remainder of the command. This response is prefixed with the token "+". Note: If instead, the server detected an error in the command, it sends a BAD completion response with a tag matching the command (as described below) to reject the command and prevent the client from sending any more of the command. It is also possible for the server to send a completion response for some other command (if multiple commands are in progress), or untagged data. In either case, the command continuation request is still pending; the client takes the appropriate action for the response, and reads another response from the server. In all cases, the client MUST send a complete command (including receiving all command continuation request responses and command continuations for the command) before initiating a new command. The protocol receiver of an IMAP4rev1 server reads a command line from the client, parses the command and its arguments, and transmits server data and a server command completion result response. 2.2.2. Server Protocol Sender and Client Protocol Receiver Data transmitted by the server to the client and status responses that do not indicate command completion are prefixed with the token "*", and are called untagged responses. Server data MAY be sent as a result of a client command, or MAY be sent unilaterally by the server. There is no syntactic difference between server data that resulted from a specific command and server data that were sent unilaterally. The server completion result response indicates the success or failure of the operation. It is tagged with the same tag as the client command which began the operation. Thus, if more than one command is in progress, the tag in a server completion response identifies the command to which the response applies. There are three possible server completion responses: OK (indicating success), NO (indicating failure), or BAD (indicating a protocol error such as unrecognized command or command syntax error). Servers SHOULD enforce the syntax outlined in this specification strictly. Any client command with a protocol syntax error, including (but not limited to) missing or extraneous spaces or arguments, Crispin Standards Track [Page 7] RFC 3501 IMAPv4 March 2003 SHOULD be rejected, and the client given a BAD server completion response. The protocol receiver of an IMAP4rev1 client reads a response line from the server. It then takes action on the response based upon the first token of the response, which can be a tag, a "*", or a "+". A client MUST be prepared to accept any server response at all times. This includes server data that was not requested. Server data SHOULD be recorded, so that the client can reference its recorded copy rather than sending a command to the server to request the data. In the case of certain server data, the data MUST be recorded. This topic is discussed in greater detail in the Server Responses section. 2.3. Message Attributes In addition to message text, each message has several attributes associated with it. These attributes can be retrieved individually or in conjunction with other attributes or message texts. 2.3.1. Message Numbers Messages in IMAP4rev1 are accessed by one of two numbers; the unique identifier or the message sequence number. 2.3.1.1. Unique Identifier (UID) Message Attribute A 32-bit value assigned to each message, which when used with the unique identifier validity value (see below) forms a 64-bit value that MUST NOT refer to any other message in the mailbox or any subsequent mailbox with the same name forever. Unique identifiers are assigned in a strictly ascending fashion in the mailbox; as each message is added to the mailbox it is assigned a higher UID than the message(s) which were added previously. Unlike message sequence numbers, unique identifiers are not necessarily contiguous. The unique identifier of a message MUST NOT change during the session, and SHOULD NOT change between sessions. Any change of unique identifiers between sessions MUST be detectable using the UIDVALIDITY mechanism discussed below. Persistent unique identifiers are required for a client to resynchronize its state from a previous session with the server (e.g., disconnected or offline access clients); this is discussed further in [IMAP-DISC]. Crispin Standards Track [Page 8] RFC 3501 IMAPv4 March 2003 Associated with every mailbox are two values which aid in unique identifier handling: the next unique identifier value and the unique identifier validity value. The next unique identifier value is the predicted value that will be assigned to a new message in the mailbox. Unless the unique identifier validity also changes (see below), the next unique identifier value MUST have the following two characteristics. First, the next unique identifier value MUST NOT change unless new messages are added to the mailbox; and second, the next unique identifier value MUST change whenever new messages are added to the mailbox, even if those new messages are subsequently expunged. Note: The next unique identifier value is intended to provide a means for a client to determine whether any messages have been delivered to the mailbox since the previous time it checked this value. It is not intended to provide any guarantee that any message will have this unique identifier. A client can only assume, at the time that it obtains the next unique identifier value, that messages arriving after that time will have a UID greater than or equal to that value. The unique identifier validity value is sent in a UIDVALIDITY response code in an OK untagged response at mailbox selection time. If unique identifiers from an earlier session fail to persist in this session, the unique identifier validity value MUST be greater than the one used in the earlier session. Note: Ideally, unique identifiers SHOULD persist at all times. Although this specification recognizes that failure to persist can be unavoidable in certain server environments, it STRONGLY ENCOURAGES message store implementation techniques that avoid this problem. For example: 1) Unique identifiers MUST be strictly ascending in the mailbox at all times. If the physical message store is re-ordered by a non-IMAP agent, this requires that the unique identifiers in the mailbox be regenerated, since the former unique identifiers are no longer strictly ascending as a result of the re-ordering. 2) If the message store has no mechanism to store unique identifiers, it must regenerate unique identifiers at each session, and each session must have a unique UIDVALIDITY value. Crispin Standards Track [Page 9] RFC 3501 IMAPv4 March 2003 3) If the mailbox is deleted and a new mailbox with the same name is created at a later date, the server must either keep track of unique identifiers from the previous instance of the mailbox, or it must assign a new UIDVALIDITY value to the new instance of the mailbox. A good UIDVALIDITY value to use in this case is a 32-bit representation of the creation date/time of the mailbox. It is alright to use a constant such as 1, but only if it guaranteed that unique identifiers will never be reused, even in the case of a mailbox being deleted (or renamed) and a new mailbox by the same name created at some future time. 4) The combination of mailbox name, UIDVALIDITY, and UID must refer to a single immutable message on that server forever. In particular, the internal date, [RFC-2822] size, envelope, body structure, and message texts (RFC822, RFC822.HEADER, RFC822.TEXT, and all BODY[...] fetch data items) must never change. This does not include message numbers, nor does it include attributes that can be set by a STORE command (e.g., FLAGS). 2.3.1.2. Message Sequence Number Message Attribute A relative position from 1 to the number of messages in the mailbox. This position MUST be ordered by ascending unique identifier. As each new message is added, it is assigned a message sequence number that is 1 higher than the number of messages in the mailbox before that new message was added. Message sequence numbers can be reassigned during the session. For example, when a message is permanently removed (expunged) from the mailbox, the message sequence number for all subsequent messages is decremented. The number of messages in the mailbox is also decremented. Similarly, a new message can be assigned a message sequence number that was once held by some other message prior to an expunge. In addition to accessing messages by relative position in the mailbox, message sequence numbers can be used in mathematical calculations. For example, if an untagged "11 EXISTS" is received, and previously an untagged "8 EXISTS" was received, three new messages have arrived with message sequence numbers of 9, 10, and 11. Another example, if message 287 in a 523 message mailbox has UID 12345, there are exactly 286 messages which have lesser UIDs and 236 messages which have greater UIDs. Crispin Standards Track [Page 10] RFC 3501 IMAPv4 March 2003 2.3.2. Flags Message Attribute A list of zero or more named tokens associated with the message. A flag is set by its addition to this list, and is cleared by its removal. There are two types of flags in IMAP4rev1. A flag of either type can be permanent or session-only. A system flag is a flag name that is pre-defined in this specification. All system flags begin with "\". Certain system flags (\Deleted and \Seen) have special semantics described elsewhere. The currently-defined system flags are: \Seen Message has been read \Answered Message has been answered \Flagged Message is "flagged" for urgent/special attention \Deleted Message is "deleted" for removal by later EXPUNGE \Draft Message has not completed composition (marked as a draft). \Recent Message is "recently" arrived in this mailbox. This session is the first session to have been notified about this message; if the session is read-write, subsequent sessions will not see \Recent set for this message. This flag can not be altered by the client. If it is not possible to determine whether or not this session is the first session to be notified about a message, then that message SHOULD be considered recent. If multiple connections have the same mailbox selected simultaneously, it is undefined which of these connections will see newly-arrived messages with \Recent set and which will see it without \Recent set. A keyword is defined by the server implementation. Keywords do not begin with "\". Servers MAY permit the client to define new keywords in the mailbox (see the description of the PERMANENTFLAGS response code for more information). Crispin Standards Track [Page 11] RFC 3501 IMAPv4 March 2003 A flag can be permanent or session-only on a per-flag basis. Permanent flags are those which the client can add or remove from the message flags permanently; that is, concurrent and subsequent sessions will see any change in permanent flags. Changes to session flags are valid only in that session. Note: The \Recent system flag is a special case of a session flag. \Recent can not be used as an argument in a STORE or APPEND command, and thus can not be changed at all. 2.3.3. Internal Date Message Attribute The internal date and time of the message on the server. This is not the date and time in the [RFC-2822] header, but rather a date and time which reflects when the message was received. In the case of messages delivered via [SMTP], this SHOULD be the date and time of final delivery of the message as defined by [SMTP]. In the case of messages delivered by the IMAP4rev1 COPY command, this SHOULD be the internal date and time of the source message. In the case of messages delivered by the IMAP4rev1 APPEND command, this SHOULD be the date and time as specified in the APPEND command description. All other cases are implementation defined. 2.3.4. [RFC-2822] Size Message Attribute The number of octets in the message, as expressed in [RFC-2822] format. 2.3.5. Envelope Structure Message Attribute A parsed representation of the [RFC-2822] header of the message. Note that the IMAP Envelope structure is not the same as an [SMTP] envelope. 2.3.6. Body Structure Message Attribute A parsed representation of the [MIME-IMB] body structure information of the message. Crispin Standards Track [Page 12] RFC 3501 IMAPv4 March 2003 2.4. Message Texts In addition to being able to fetch the full [RFC-2822] text of a message, IMAP4rev1 permits the fetching of portions of the full message text. Specifically, it is possible to fetch the [RFC-2822] message header, [RFC-2822] message body, a [MIME-IMB] body part, or a [MIME-IMB] header. 3. State and Flow Diagram Once the connection between client and server is established, an IMAP4rev1 connection is in one of four states. The initial state is identified in the server greeting. Most commands are only valid in certain states. It is a protocol error for the client to attempt a command while the connection is in an inappropriate state, and the server will respond with a BAD or NO (depending upon server implementation) command completion result. 3.1. Not Authenticated State In the not authenticated state, the client MUST supply authentication credentials before most commands will be permitted. This state is entered when a connection starts unless the connection has been pre-authenticated. 3.2. Authenticated State In the authenticated state, the client is authenticated and MUST select a mailbox to access before commands that affect messages will be permitted. This state is entered when a pre-authenticated connection starts, when acceptable authentication credentials have been provided, after an error in selecting a mailbox, or after a successful CLOSE command. 3.3. Selected State In a selected state, a mailbox has been selected to access. This state is entered when a mailbox has been successfully selected. Crispin Standards Track [Page 13] RFC 3501 IMAPv4 March 2003 3.4. Logout State In the logout state, the connection is being terminated. This state can be entered as a result of a client request (via the LOGOUT command) or by unilateral action on the part of either the client or server. If the client requests the logout state, the server MUST send an untagged BYE response and a tagged OK response to the LOGOUT command before the server closes the connection; and the client MUST read the tagged OK response to the LOGOUT command before the client closes the connection. A server MUST NOT unilaterally close the connection without sending an untagged BYE response that contains the reason for having done so. A client SHOULD NOT unilaterally close the connection, and instead SHOULD issue a LOGOUT command. If the server detects that the client has unilaterally closed the connection, the server MAY omit the untagged BYE response and simply close its connection. Crispin Standards Track [Page 14] RFC 3501 IMAPv4 March 2003 +----------------------+ |connection established| +----------------------+ || \/ +--------------------------------------+ | server greeting | +--------------------------------------+ || (1) || (2) || (3) \/ || || +-----------------+ || || |Not Authenticated| || || +-----------------+ || || || (7) || (4) || || || \/ \/ || || +----------------+ || || | Authenticated |<=++ || || +----------------+ || || || || (7) || (5) || (6) || || || \/ || || || || +--------+ || || || || |Selected|==++ || || || +--------+ || || || || (7) || \/ \/ \/ \/ +--------------------------------------+ | Logout | +--------------------------------------+ || \/ +-------------------------------+ |both sides close the connection| +-------------------------------+ (1) connection without pre-authentication (OK greeting) (2) pre-authenticated connection (PREAUTH greeting) (3) rejected connection (BYE greeting) (4) successful LOGIN or AUTHENTICATE command (5) successful SELECT or EXAMINE command (6) CLOSE command, or failed SELECT or EXAMINE command (7) LOGOUT command, server shutdown, or connection closed Crispin Standards Track [Page 15] RFC 3501 IMAPv4 March 2003 4. Data Formats IMAP4rev1 uses textual commands and responses. Data in IMAP4rev1 can be in one of several forms: atom, number, string, parenthesized list, or NIL. Note that a particular data item may take more than one form; for example, a data item defined as using "astring" syntax may be either an atom or a string. 4.1. Atom An atom consists of one or more non-special characters. 4.2. Number A number consists of one or more digit characters, and represents a numeric value. 4.3. String A string is in one of two forms: either literal or quoted string. The literal form is the general form of string. The quoted string form is an alternative that avoids the overhead of processing a literal at the cost of limitations of characters which may be used. A literal is a sequence of zero or more octets (including CR and LF), prefix-quoted with an octet count in the form of an open brace ("{"), the number of octets, close brace ("}"), and CRLF. In the case of literals transmitted from server to client, the CRLF is immediately followed by the octet data. In the case of literals transmitted from client to server, the client MUST wait to receive a command continuation request (described later in this document) before sending the octet data (and the remainder of the command). A quoted string is a sequence of zero or more 7-bit characters, excluding CR and LF, with double quote (<">) characters at each end. The empty string is represented as either "" (a quoted string with zero characters between double quotes) or as {0} followed by CRLF (a literal with an octet count of 0). Note: Even if the octet count is 0, a client transmitting a literal MUST wait to receive a command continuation request. Crispin Standards Track [Page 16] RFC 3501 IMAPv4 March 2003 4.3.1. 8-bit and Binary Strings 8-bit textual and binary mail is supported through the use of a [MIME-IMB] content transfer encoding. IMAP4rev1 implementations MAY transmit 8-bit or multi-octet characters in literals, but SHOULD do so only when the [CHARSET] is identified. Although a BINARY body encoding is defined, unencoded binary strings are not permitted. A "binary string" is any string with NUL characters. Implementations MUST encode binary data into a textual form, such as BASE64, before transmitting the data. A string with an excessive amount of CTL characters MAY also be considered to be binary. 4.4. Parenthesized List Data structures are represented as a "parenthesized list"; a sequence of data items, delimited by space, and bounded at each end by parentheses. A parenthesized list can contain other parenthesized lists, using multiple levels of parentheses to indicate nesting. The empty list is represented as () -- a parenthesized list with no members. 4.5. NIL The special form "NIL" represents the non-existence of a particular data item that is represented as a string or parenthesized list, as distinct from the empty string "" or the empty parenthesized list (). Note: NIL is never used for any data item which takes the form of an atom. For example, a mailbox name of "NIL" is a mailbox named NIL as opposed to a non-existent mailbox name. This is because mailbox uses "astring" syntax which is an atom or a string. Conversely, an addr-name of NIL is a non-existent personal name, because addr-name uses "nstring" syntax which is NIL or a string, but never an atom. Crispin Standards Track [Page 17] RFC 3501 IMAPv4 March 2003 5. Operational Considerations The following rules are listed here to ensure that all IMAP4rev1 implementations interoperate properly. 5.1. Mailbox Naming Mailbox names are 7-bit. Client implementations MUST NOT attempt to create 8-bit mailbox names, and SHOULD interpret any 8-bit mailbox names returned by LIST or LSUB as UTF-8. Server implementations SHOULD prohibit the creation of 8-bit mailbox names, and SHOULD NOT return 8-bit mailbox names in LIST or LSUB. See section 5.1.3 for more information on how to represent non-ASCII mailbox names. Note: 8-bit mailbox names were undefined in earlier versions of this protocol. Some sites used a local 8-bit character set to represent non-ASCII mailbox names. Such usage is not interoperable, and is now formally deprecated. The case-insensitive mailbox name INBOX is a special name reserved to mean "the primary mailbox for this user on this server". The interpretation of all other names is implementation-dependent. In particular, this specification takes no position on case sensitivity in non-INBOX mailbox names. Some server implementations are fully case-sensitive; others preserve case of a newly-created name but otherwise are case-insensitive; and yet others coerce names to a particular case. Client implementations MUST interact with any of these. If a server implementation interprets non-INBOX mailbox names as case-insensitive, it MUST treat names using the international naming convention specially as described in section 5.1.3. There are certain client considerations when creating a new mailbox name: 1) Any character which is one of the atom-specials (see the Formal Syntax) will require that the mailbox name be represented as a quoted string or literal. 2) CTL and other non-graphic characters are difficult to represent in a user interface and are best avoided. 3) Although the list-wildcard characters ("%" and "*") are valid in a mailbox name, it is difficult to use such mailbox names with the LIST and LSUB commands due to the conflict with wildcard interpretation. Crispin Standards Track [Page 18] RFC 3501 IMAPv4 March 2003 4) Usually, a character (determined by the server implementation) is reserved to delimit levels of hierarchy. 5) Two characters, "#" and "&", have meanings by convention, and should be avoided except when used in that convention. 5.1.1. Mailbox Hierarchy Naming If it is desired to export hierarchical mailbox names, mailbox names MUST be left-to-right hierarchical using a single character to separate levels of hierarchy. The same hierarchy separator character is used for all levels of hierarchy within a single name. 5.1.2. Mailbox Namespace Naming Convention By convention, the first hierarchical element of any mailbox name which begins with "#" identifies the "namespace" of the remainder of the name. This makes it possible to disambiguate between different types of mailbox stores, each of which have their own namespaces. For example, implementations which offer access to USENET newsgroups MAY use the "#news" namespace to partition the USENET newsgroup namespace from that of other mailboxes. Thus, the comp.mail.misc newsgroup would have a mailbox name of "#news.comp.mail.misc", and the name "comp.mail.misc" can refer to a different object (e.g., a user's private mailbox). 5.1.3. Mailbox International Naming Convention By convention, international mailbox names in IMAP4rev1 are specified using a modified version of the UTF-7 encoding described in [UTF-7]. Modified UTF-7 may also be usable in servers that implement an earlier version of this protocol. In modified UTF-7, printable US-ASCII characters, except for "&", represent themselves; that is, characters with octet values 0x20-0x25 and 0x27-0x7e. The character "&" (0x26) is represented by the two-octet sequence "&-". All other characters (octet values 0x00-0x1f and 0x7f-0xff) are represented in modified BASE64, with a further modification from [UTF-7] that "," is used instead of "/". Modified BASE64 MUST NOT be used to represent any printing US-ASCII character which can represent itself. Crispin Standards Track [Page 19] RFC 3501 IMAPv4 March 2003 "&" is used to shift to modified BASE64 and "-" to shift back to US-ASCII. There is no implicit shift from BASE64 to US-ASCII, and null shifts ("-&" while in BASE64; note that "&-" while in US-ASCII means "&") are not permitted. However, all names start in US-ASCII, and MUST end in US-ASCII; that is, a name that ends with a non-ASCII ISO-10646 character MUST end with a "-"). The purpose of these modifications is to correct the following problems with UTF-7: 1) UTF-7 uses the "+" character for shifting; this conflicts with the common use of "+" in mailbox names, in particular USENET newsgroup names. 2) UTF-7's encoding is BASE64 which uses the "/" character; this conflicts with the use of "/" as a popular hierarchy delimiter. 3) UTF-7 prohibits the unencoded usage of "\"; this conflicts with the use of "\" as a popular hierarchy delimiter. 4) UTF-7 prohibits the unencoded usage of "~"; this conflicts with the use of "~" in some servers as a home directory indicator. 5) UTF-7 permits multiple alternate forms to represent the same string; in particular, printable US-ASCII characters can be represented in encoded form. Although modified UTF-7 is a convention, it establishes certain requirements on server handling of any mailbox name with an embedded "&" character. In particular, server implementations MUST preserve the exact form of the modified BASE64 portion of a modified UTF-7 name and treat that text as case-sensitive, even if names are otherwise case-insensitive or case-folded. Server implementations SHOULD verify that any mailbox name with an embedded "&" character, used as an argument to CREATE, is: in the correctly modified UTF-7 syntax, has no superfluous shifts, and has no encoding in modified BASE64 of any printing US-ASCII character which can represent itself. However, client implementations MUST NOT depend upon the server doing this, and SHOULD NOT attempt to create a mailbox name with an embedded "&" character unless it complies with the modified UTF-7 syntax. Server implementations which export a mail store that does not follow the modified UTF-7 convention MUST convert to modified UTF-7 any mailbox name that contains either non-ASCII characters or the "&" character. Crispin Standards Track [Page 20] RFC 3501 IMAPv4 March 2003 For example, here is a mailbox name which mixes English, Chinese, and Japanese text: ~peter/mail/&U,BTFw-/&ZeVnLIqe- For example, the string "&Jjo!" is not a valid mailbox name because it does not contain a shift to US-ASCII before the "!". The correct form is "&Jjo-!". The string "&U,BTFw-&ZeVnLIqe-" is not permitted because it contains a superfluous shift. The correct form is "&U,BTF2XlZyyKng-". 5.2. Mailbox Size and Message Status Updates At any time, a server can send data that the client did not request. Sometimes, such behavior is REQUIRED. For example, agents other than the server MAY add messages to the mailbox (e.g., new message delivery), change the flags of the messages in the mailbox (e.g., simultaneous access to the same mailbox by multiple agents), or even remove messages from the mailbox. A server MUST send mailbox size updates automatically if a mailbox size change is observed during the processing of a command. A server SHOULD send message flag updates automatically, without requiring the client to request such updates explicitly. Special rules exist for server notification of a client about the removal of messages to prevent synchronization errors; see the description of the EXPUNGE response for more detail. In particular, it is NOT permitted to send an EXISTS response that would reduce the number of messages in the mailbox; only the EXPUNGE response can do this. Regardless of what implementation decisions a client makes on remembering data from the server, a client implementation MUST record mailbox size updates. It MUST NOT assume that any command after the initial mailbox selection will return the size of the mailbox. 5.3. Response when no Command in Progress Server implementations are permitted to send an untagged response (except for EXPUNGE) while there is no command in progress. Server implementations that send such responses MUST deal with flow control considerations. Specifically, they MUST either (1) verify that the size of the data does not exceed the underlying transport's available window size, or (2) use non-blocking writes. Crispin Standards Track [Page 21] RFC 3501 IMAPv4 March 2003 5.4. Autologout Timer If a server has an inactivity autologout timer, the duration of that timer MUST be at least 30 minutes. The receipt of ANY command from the client during that interval SHOULD suffice to reset the autologout timer. 5.5. Multiple Commands in Progress The client MAY send another command without waiting for the completion result response of a command, subject to ambiguity rules (see below) and flow control constraints on the underlying data stream. Similarly, a server MAY begin processing another command before processing the current command to completion, subject to ambiguity rules. However, any command continuation request responses and command continuations MUST be negotiated before any subsequent command is initiated. The exception is if an ambiguity would result because of a command that would affect the results of other commands. Clients MUST NOT send multiple commands without waiting if an ambiguity would result. If the server detects a possible ambiguity, it MUST execute commands to completion in the order given by the client. The most obvious example of ambiguity is when a command would affect the results of another command, e.g., a FETCH of a message's flags and a STORE of that same message's flags. A non-obvious ambiguity occurs with commands that permit an untagged EXPUNGE response (commands other than FETCH, STORE, and SEARCH), since an untagged EXPUNGE response can invalidate sequence numbers in a subsequent command. This is not a problem for FETCH, STORE, or SEARCH commands because servers are prohibited from sending EXPUNGE responses while any of those commands are in progress. Therefore, if the client sends any command other than FETCH, STORE, or SEARCH, it MUST wait for the completion result response before sending a command with message sequence numbers. Note: UID FETCH, UID STORE, and UID SEARCH are different commands from FETCH, STORE, and SEARCH. If the client sends a UID command, it must wait for a completion result response before sending a command with message sequence numbers. Crispin Standards Track [Page 22] RFC 3501 IMAPv4 March 2003 For example, the following non-waiting command sequences are invalid: FETCH + NOOP + STORE STORE + COPY + FETCH COPY + COPY CHECK + FETCH The following are examples of valid non-waiting command sequences: FETCH + STORE + SEARCH + CHECK STORE + COPY + EXPUNGE UID SEARCH + UID SEARCH may be valid or invalid as a non-waiting command sequence, depending upon whether or not the second UID SEARCH contains message sequence numbers. 6. Client Commands IMAP4rev1 commands are described in this section. Commands are organized by the state in which the command is permitted. Commands which are permitted in multiple states are listed in the minimum permitted state (for example, commands valid in authenticated and selected state are listed in the authenticated state commands). Command arguments, identified by "Arguments:" in the command descriptions below, are described by function, not by syntax. The precise syntax of command arguments is described in the Formal Syntax section. Some commands cause specific server responses to be returned; these are identified by "Responses:" in the command descriptions below. See the response descriptions in the Responses section for information on these responses, and the Formal Syntax section for the precise syntax of these responses. It is possible for server data to be transmitted as a result of any command. Thus, commands that do not specifically require server data specify "no specific responses for this command" instead of "none". The "Result:" in the command description refers to the possible tagged status responses to a command, and any special interpretation of these status responses. The state of a connection is only changed by successful commands which are documented as changing state. A rejected command (BAD response) never changes the state of the connection or of the selected mailbox. A failed command (NO response) generally does not change the state of the connection or of the selected mailbox; the exception being the SELECT and EXAMINE commands. Crispin Standards Track [Page 23] RFC 3501 IMAPv4 March 2003 6.1. Client Commands - Any State The following commands are valid in any state: CAPABILITY, NOOP, and LOGOUT. 6.1.1. CAPABILITY Command Arguments: none Responses: REQUIRED untagged response: CAPABILITY Result: OK - capability completed BAD - command unknown or arguments invalid The CAPABILITY command requests a listing of capabilities that the server supports. The server MUST send a single untagged CAPABILITY response with "IMAP4rev1" as one of the listed capabilities before the (tagged) OK response. A capability name which begins with "AUTH=" indicates that the server supports that particular authentication mechanism. All such names are, by definition, part of this specification. For example, the authorization capability for an experimental "blurdybloop" authenticator would be "AUTH=XBLURDYBLOOP" and not "XAUTH=BLURDYBLOOP" or "XAUTH=XBLURDYBLOOP". Other capability names refer to extensions, revisions, or amendments to this specification. See the documentation of the CAPABILITY response for additional information. No capabilities, beyond the base IMAP4rev1 set defined in this specification, are enabled without explicit client action to invoke the capability. Client and server implementations MUST implement the STARTTLS, LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) capabilities. See the Security Considerations section for important information. See the section entitled "Client Commands - Experimental/Expansion" for information about the form of site or implementation-specific capabilities. Crispin Standards Track [Page 24] RFC 3501 IMAPv4 March 2003 Example: C: abcd CAPABILITY S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI LOGINDISABLED S: abcd OK CAPABILITY completed C: efgh STARTTLS S: efgh OK STARTLS completed C: ijkl CAPABILITY S: * CAPABILITY IMAP4rev1 AUTH=GSSAPI AUTH=PLAIN S: ijkl OK CAPABILITY completed 6.1.2. NOOP Command Arguments: none Responses: no specific responses for this command (but see below) Result: OK - noop completed BAD - command unknown or arguments invalid The NOOP command always succeeds. It does nothing. Since any command can return a status update as untagged data, the NOOP command can be used as a periodic poll for new messages or message status updates during a period of inactivity (this is the preferred method to do this). The NOOP command can also be used to reset any inactivity autologout timer on the server. Example: C: a002 NOOP S: a002 OK NOOP completed . . . C: a047 NOOP S: * 22 EXPUNGE S: * 23 EXISTS S: * 3 RECENT S: * 14 FETCH (FLAGS (\Seen \Deleted)) S: a047 OK NOOP completed Crispin Standards Track [Page 25] RFC 3501 IMAPv4 March 2003 6.1.3. LOGOUT Command Arguments: none Responses: REQUIRED untagged response: BYE Result: OK - logout completed BAD - command unknown or arguments invalid The LOGOUT command informs the server that the client is done with the connection. The server MUST send a BYE untagged response before the (tagged) OK response, and then close the network connection. Example: C: A023 LOGOUT S: * BYE IMAP4rev1 Server logging out S: A023 OK LOGOUT completed (Server and client then close the connection) 6.2. Client Commands - Not Authenticated State In the not authenticated state, the AUTHENTICATE or LOGIN command establishes authentication and enters the authenticated state. The AUTHENTICATE command provides a general mechanism for a variety of authentication techniques, privacy protection, and integrity checking; whereas the LOGIN command uses a traditional user name and plaintext password pair and has no means of establishing privacy protection or integrity checking. The STARTTLS command is an alternate form of establishing session privacy protection and integrity checking, but does not establish authentication or enter the authenticated state. Server implementations MAY allow access to certain mailboxes without establishing authentication. This can be done by means of the ANONYMOUS [SASL] authenticator described in [ANONYMOUS]. An older convention is a LOGIN command using the userid "anonymous"; in this case, a password is required although the server may choose to accept any password. The restrictions placed on anonymous users are implementation-dependent. Once authenticated (including as anonymous), it is not possible to re-enter not authenticated state. Crispin Standards Track [Page 26] RFC 3501 IMAPv4 March 2003 In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), the following commands are valid in the not authenticated state: STARTTLS, AUTHENTICATE and LOGIN. See the Security Considerations section for important information about these commands. 6.2.1. STARTTLS Command Arguments: none Responses: no specific response for this command Result: OK - starttls completed, begin TLS negotiation BAD - command unknown or arguments invalid A [TLS] negotiation begins immediately after the CRLF at the end of the tagged OK response from the server. Once a client issues a STARTTLS command, it MUST NOT issue further commands until a server response is seen and the [TLS] negotiation is complete. The server remains in the non-authenticated state, even if client credentials are supplied during the [TLS] negotiation. This does not preclude an authentication mechanism such as EXTERNAL (defined in [SASL]) from using client identity determined by the [TLS] negotiation. Once [TLS] has been started, the client MUST discard cached information about server capabilities and SHOULD re-issue the CAPABILITY command. This is necessary to protect against man-in- the-middle attacks which alter the capabilities list prior to STARTTLS. The server MAY advertise different capabilities after STARTTLS. Example: C: a001 CAPABILITY S: * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED S: a001 OK CAPABILITY completed C: a002 STARTTLS S: a002 OK Begin TLS negotiation now C: a003 CAPABILITY S: * CAPABILITY IMAP4rev1 AUTH=PLAIN S: a003 OK CAPABILITY completed C: a004 LOGIN joe password S: a004 OK LOGIN completed Crispin Standards Track [Page 27] RFC 3501 IMAPv4 March 2003 6.2.2. AUTHENTICATE Command Arguments: authentication mechanism name Responses: continuation data can be requested Result: OK - authenticate completed, now in authenticated state NO - authenticate failure: unsupported authentication mechanism, credentials rejected BAD - command unknown or arguments invalid, authentication exchange cancelled The AUTHENTICATE command indicates a [SASL] authentication mechanism to the server. If the server supports the requested authentication mechanism, it performs an authentication protocol exchange to authenticate and identify the client. It MAY also negotiate an OPTIONAL security layer for subsequent protocol interactions. If the requested authentication mechanism is not supported, the server SHOULD reject the AUTHENTICATE command by sending a tagged NO response. The AUTHENTICATE command does not support the optional "initial response" feature of [SASL]. Section 5.1 of [SASL] specifies how to handle an authentication mechanism which uses an initial response. The service name specified by this protocol's profile of [SASL] is "imap". The authentication protocol exchange consists of a series of server challenges and client responses that are specific to the authentication mechanism. A server challenge consists of a command continuation request response with the "+" token followed by a BASE64 encoded string. The client response consists of a single line consisting of a BASE64 encoded string. If the client wishes to cancel an authentication exchange, it issues a line consisting of a single "*". If the server receives such a response, it MUST reject the AUTHENTICATE command by sending a tagged BAD response. If a security layer is negotiated through the [SASL] authentication exchange, it takes effect immediately following the CRLF that concludes the authentication exchange for the client, and the CRLF of the tagged OK response for the server. While client and server implementations MUST implement the AUTHENTICATE command itself, it is not required to implement any authentication mechanisms other than the PLAIN mechanism described Crispin Standards Track [Page 28] RFC 3501 IMAPv4 March 2003 in [IMAP-TLS]. Also, an authentication mechanism is not required to support any security layers. Note: a server implementation MUST implement a configuration in which it does NOT permit any plaintext password mechanisms, unless either the STARTTLS command has been negotiated or some other mechanism that protects the session from password snooping has been provided. Server sites SHOULD NOT use any configuration which permits a plaintext password mechanism without such a protection mechanism against password snooping. Client and server implementations SHOULD implement additional [SASL] mechanisms that do not use plaintext passwords, such the GSSAPI mechanism described in [SASL] and/or the [DIGEST-MD5] mechanism. Servers and clients can support multiple authentication mechanisms. The server SHOULD list its supported authentication mechanisms in the response to the CAPABILITY command so that the client knows which authentication mechanisms to use. A server MAY include a CAPABILITY response code in the tagged OK response of a successful AUTHENTICATE command in order to send capabilities automatically. It is unnecessary for a client to send a separate CAPABILITY command if it recognizes these automatic capabilities. This should only be done if a security layer was not negotiated by the AUTHENTICATE command, because the tagged OK response as part of an AUTHENTICATE command is not protected by encryption/integrity checking. [SASL] requires the client to re-issue a CAPABILITY command in this case. If an AUTHENTICATE command fails with a NO response, the client MAY try another authentication mechanism by issuing another AUTHENTICATE command. It MAY also attempt to authenticate by using the LOGIN command (see section 6.2.3 for more detail). In other words, the client MAY request authentication types in decreasing order of preference, with the LOGIN command as a last resort. The authorization identity passed from the client to the server during the authentication exchange is interpreted by the server as the user name whose privileges the client is requesting. Crispin Standards Track [Page 29] RFC 3501 IMAPv4 March 2003 Example: S: * OK IMAP4rev1 Server C: A001 AUTHENTICATE GSSAPI S: + C: YIIB+wYJKoZIhvcSAQICAQBuggHqMIIB5qADAgEFoQMCAQ6iBw MFACAAAACjggEmYYIBIjCCAR6gAwIBBaESGxB1Lndhc2hpbmd0 b24uZWR1oi0wK6ADAgEDoSQwIhsEaW1hcBsac2hpdmFtcy5jYW Mud2FzaGluZ3Rvbi5lZHWjgdMwgdCgAwIBAaEDAgEDooHDBIHA cS1GSa5b+fXnPZNmXB9SjL8Ollj2SKyb+3S0iXMljen/jNkpJX AleKTz6BQPzj8duz8EtoOuNfKgweViyn/9B9bccy1uuAE2HI0y C/PHXNNU9ZrBziJ8Lm0tTNc98kUpjXnHZhsMcz5Mx2GR6dGknb I0iaGcRerMUsWOuBmKKKRmVMMdR9T3EZdpqsBd7jZCNMWotjhi vd5zovQlFqQ2Wjc2+y46vKP/iXxWIuQJuDiisyXF0Y8+5GTpAL pHDc1/pIGmMIGjoAMCAQGigZsEgZg2on5mSuxoDHEA1w9bcW9n FdFxDKpdrQhVGVRDIzcCMCTzvUboqb5KjY1NJKJsfjRQiBYBdE NKfzK+g5DlV8nrw81uOcP8NOQCLR5XkoMHC0Dr/80ziQzbNqhx O6652Npft0LQwJvenwDI13YxpwOdMXzkWZN/XrEqOWp6GCgXTB vCyLWLlWnbaUkZdEYbKHBPjd8t/1x5Yg== S: + YGgGCSqGSIb3EgECAgIAb1kwV6ADAgEFoQMCAQ+iSzBJoAMC AQGiQgRAtHTEuOP2BXb9sBYFR4SJlDZxmg39IxmRBOhXRKdDA0 uHTCOT9Bq3OsUTXUlk0CsFLoa8j+gvGDlgHuqzWHPSQg== C: S: + YDMGCSqGSIb3EgECAgIBAAD/////6jcyG4GE3KkTzBeBiVHe ceP2CWY0SR0fAQAgAAQEBAQ= C: YDMGCSqGSIb3EgECAgIBAAD/////3LQBHXTpFfZgrejpLlLImP wkhbfa2QteAQAgAG1yYwE= S: A001 OK GSSAPI authentication successful Note: The line breaks within server challenges and client responses are for editorial clarity and are not in real authenticators. 6.2.3. LOGIN Command Arguments: user name password Responses: no specific responses for this command Result: OK - login completed, now in authenticated state NO - login failure: user name or password rejected BAD - command unknown or arguments invalid The LOGIN command identifies the client to the server and carries the plaintext password authenticating this user. Crispin Standards Track [Page 30] RFC 3501 IMAPv4 March 2003 A server MAY include a CAPABILITY response code in the tagged OK response to a successful LOGIN command in order to send capabilities automatically. It is unnecessary for a client to send a separate CAPABILITY command if it recognizes these automatic capabilities. Example: C: a001 LOGIN SMITH SESAME S: a001 OK LOGIN completed Note: Use of the LOGIN command over an insecure network (such as the Internet) is a security risk, because anyone monitoring network traffic can obtain plaintext passwords. The LOGIN command SHOULD NOT be used except as a last resort, and it is recommended that client implementations have a means to disable any automatic use of the LOGIN command. Unless either the STARTTLS command has been negotiated or some other mechanism that protects the session from password snooping has been provided, a server implementation MUST implement a configuration in which it advertises the LOGINDISABLED capability and does NOT permit the LOGIN command. Server sites SHOULD NOT use any configuration which permits the LOGIN command without such a protection mechanism against password snooping. A client implementation MUST NOT send a LOGIN command if the LOGINDISABLED capability is advertised. 6.3. Client Commands - Authenticated State In the authenticated state, commands that manipulate mailboxes as atomic entities are permitted. Of these commands, the SELECT and EXAMINE commands will select a mailbox for access and enter the selected state. In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), the following commands are valid in the authenticated state: SELECT, EXAMINE, CREATE, DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, STATUS, and APPEND. Crispin Standards Track [Page 31] RFC 3501 IMAPv4 March 2003 6.3.1. SELECT Command Arguments: mailbox name Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, UIDNEXT, UIDVALIDITY Result: OK - select completed, now in selected state NO - select failure, now in authenticated state: no such mailbox, can't access mailbox BAD - command unknown or arguments invalid The SELECT command selects a mailbox so that messages in the mailbox can be accessed. Before returning an OK to the client, the server MUST send the following untagged data to the client. Note that earlier versions of this protocol only required the FLAGS, EXISTS, and RECENT untagged data; consequently, client implementations SHOULD implement default behavior for missing data as discussed with the individual item. FLAGS Defined flags in the mailbox. See the description of the FLAGS response for more detail. EXISTS The number of messages in the mailbox. See the description of the EXISTS response for more detail. RECENT The number of messages with the \Recent flag set. See the description of the RECENT response for more detail. OK [UNSEEN ] The message sequence number of the first unseen message in the mailbox. If this is missing, the client can not make any assumptions about the first unseen message in the mailbox, and needs to issue a SEARCH command if it wants to find it. OK [PERMANENTFLAGS ()] A list of message flags that the client can change permanently. If this is missing, the client should assume that all flags can be changed permanently. OK [UIDNEXT ] The next unique identifier value. Refer to section 2.3.1.1 for more information. If this is missing, the client can not make any assumptions about the next unique identifier value. Crispin Standards Track [Page 32] RFC 3501 IMAPv4 March 2003 OK [UIDVALIDITY ] The unique identifier validity value. Refer to section 2.3.1.1 for more information. If this is missing, the server does not support unique identifiers. Only one mailbox can be selected at a time in a connection; simultaneous access to multiple mailboxes requires multiple connections. The SELECT command automatically deselects any currently selected mailbox before attempting the new selection. Consequently, if a mailbox is selected and a SELECT command that fails is attempted, no mailbox is selected. If the client is permitted to modify the mailbox, the server SHOULD prefix the text of the tagged OK response with the "[READ-WRITE]" response code. If the client is not permitted to modify the mailbox but is permitted read access, the mailbox is selected as read-only, and the server MUST prefix the text of the tagged OK response to SELECT with the "[READ-ONLY]" response code. Read-only access through SELECT differs from the EXAMINE command in that certain read-only mailboxes MAY permit the change of permanent state on a per-user (as opposed to global) basis. Netnews messages marked in a server-based .newsrc file are an example of such per-user permanent state that can be modified with read-only mailboxes. Example: C: A142 SELECT INBOX S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: A142 OK [READ-WRITE] SELECT completed Crispin Standards Track [Page 33] RFC 3501 IMAPv4 March 2003 6.3.2. EXAMINE Command Arguments: mailbox name Responses: REQUIRED untagged responses: FLAGS, EXISTS, RECENT REQUIRED OK untagged responses: UNSEEN, PERMANENTFLAGS, UIDNEXT, UIDVALIDITY Result: OK - examine completed, now in selected state NO - examine failure, now in authenticated state: no such mailbox, can't access mailbox BAD - command unknown or arguments invalid The EXAMINE command is identical to SELECT and returns the same output; however, the selected mailbox is identified as read-only. No changes to the permanent state of the mailbox, including per-user state, are permitted; in particular, EXAMINE MUST NOT cause messages to lose the \Recent flag. The text of the tagged OK response to the EXAMINE command MUST begin with the "[READ-ONLY]" response code. Example: C: A932 EXAMINE blurdybloop S: * 17 EXISTS S: * 2 RECENT S: * OK [UNSEEN 8] Message 8 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS ()] No permanent flags permitted S: A932 OK [READ-ONLY] EXAMINE completed 6.3.3. CREATE Command Arguments: mailbox name Responses: no specific responses for this command Result: OK - create completed NO - create failure: can't create mailbox with that name BAD - command unknown or arguments invalid The CREATE command creates a mailbox with the given name. An OK response is returned only if a new mailbox with that name has been created. It is an error to attempt to create INBOX or a mailbox with a name that refers to an extant mailbox. Any error in creation will return a tagged NO response. Crispin Standards Track [Page 34] RFC 3501 IMAPv4 March 2003 If the mailbox name is suffixed with the server's hierarchy separator character (as returned from the server by a LIST command), this is a declaration that the client intends to create mailbox names under this name in the hierarchy. Server implementations that do not require this declaration MUST ignore the declaration. In any case, the name created is without the trailing hierarchy delimiter. If the server's hierarchy separator character appears elsewhere in the name, the server SHOULD create any superior hierarchical names that are needed for the CREATE command to be successfully completed. In other words, an attempt to create "foo/bar/zap" on a server in which "/" is the hierarchy separator character SHOULD create foo/ and foo/bar/ if they do not already exist. If a new mailbox is created with the same name as a mailbox which was deleted, its unique identifiers MUST be greater than any unique identifiers used in the previous incarnation of the mailbox UNLESS the new incarnation has a different unique identifier validity value. See the description of the UID command for more detail. Example: C: A003 CREATE owatagusiam/ S: A003 OK CREATE completed C: A004 CREATE owatagusiam/blurdybloop S: A004 OK CREATE completed Note: The interpretation of this example depends on whether "/" was returned as the hierarchy separator from LIST. If "/" is the hierarchy separator, a new level of hierarchy named "owatagusiam" with a member called "blurdybloop" is created. Otherwise, two mailboxes at the same hierarchy level are created. 6.3.4. DELETE Command Arguments: mailbox name Responses: no specific responses for this command Result: OK - delete completed NO - delete failure: can't delete mailbox with that name BAD - command unknown or arguments invalid Crispin Standards Track [Page 35] RFC 3501 IMAPv4 March 2003 The DELETE command permanently removes the mailbox with the given name. A tagged OK response is returned only if the mailbox has been deleted. It is an error to attempt to delete INBOX or a mailbox name that does not exist. The DELETE command MUST NOT remove inferior hierarchical names. For example, if a mailbox "foo" has an inferior "foo.bar" (assuming "." is the hierarchy delimiter character), removing "foo" MUST NOT remove "foo.bar". It is an error to attempt to delete a name that has inferior hierarchical names and also has the \Noselect mailbox name attribute (see the description of the LIST response for more details). It is permitted to delete a name that has inferior hierarchical names and does not have the \Noselect mailbox name attribute. In this case, all messages in that mailbox are removed, and the name will acquire the \Noselect mailbox name attribute. The value of the highest-used unique identifier of the deleted mailbox MUST be preserved so that a new mailbox created with the same name will not reuse the identifiers of the former incarnation, UNLESS the new incarnation has a different unique identifier validity value. See the description of the UID command for more detail. Examples: C: A682 LIST "" * S: * LIST () "/" blurdybloop S: * LIST (\Noselect) "/" foo S: * LIST () "/" foo/bar S: A682 OK LIST completed C: A683 DELETE blurdybloop S: A683 OK DELETE completed C: A684 DELETE foo S: A684 NO Name "foo" has inferior hierarchical names C: A685 DELETE foo/bar S: A685 OK DELETE Completed C: A686 LIST "" * S: * LIST (\Noselect) "/" foo S: A686 OK LIST completed C: A687 DELETE foo S: A687 OK DELETE Completed Crispin Standards Track [Page 36] RFC 3501 IMAPv4 March 2003 C: A82 LIST "" * S: * LIST () "." blurdybloop S: * LIST () "." foo S: * LIST () "." foo.bar S: A82 OK LIST completed C: A83 DELETE blurdybloop S: A83 OK DELETE completed C: A84 DELETE foo S: A84 OK DELETE Completed C: A85 LIST "" * S: * LIST () "." foo.bar S: A85 OK LIST completed C: A86 LIST "" % S: * LIST (\Noselect) "." foo S: A86 OK LIST completed 6.3.5. RENAME Command Arguments: existing mailbox name new mailbox name Responses: no specific responses for this command Result: OK - rename completed NO - rename failure: can't rename mailbox with that name, can't rename to mailbox with that name BAD - command unknown or arguments invalid The RENAME command changes the name of a mailbox. A tagged OK response is returned only if the mailbox has been renamed. It is an error to attempt to rename from a mailbox name that does not exist or to a mailbox name that already exists. Any error in renaming will return a tagged NO response. If the name has inferior hierarchical names, then the inferior hierarchical names MUST also be renamed. For example, a rename of "foo" to "zap" will rename "foo/bar" (assuming "/" is the hierarchy delimiter character) to "zap/bar". If the server's hierarchy separator character appears in the name, the server SHOULD create any superior hierarchical names that are needed for the RENAME command to complete successfully. In other words, an attempt to rename "foo/bar/zap" to baz/rag/zowie on a server in which "/" is the hierarchy separator character SHOULD create baz/ and baz/rag/ if they do not already exist. Crispin Standards Track [Page 37] RFC 3501 IMAPv4 March 2003 The value of the highest-used unique identifier of the old mailbox name MUST be preserved so that a new mailbox created with the same name will not reuse the identifiers of the former incarnation, UNLESS the new incarnation has a different unique identifier validity value. See the description of the UID command for more detail. Renaming INBOX is permitted, and has special behavior. It moves all messages in INBOX to a new mailbox with the given name, leaving INBOX empty. If the server implementation supports inferior hierarchical names of INBOX, these are unaffected by a rename of INBOX. Examples: C: A682 LIST "" * S: * LIST () "/" blurdybloop S: * LIST (\Noselect) "/" foo S: * LIST () "/" foo/bar S: A682 OK LIST completed C: A683 RENAME blurdybloop sarasoop S: A683 OK RENAME completed C: A684 RENAME foo zowie S: A684 OK RENAME Completed C: A685 LIST "" * S: * LIST () "/" sarasoop S: * LIST (\Noselect) "/" zowie S: * LIST () "/" zowie/bar S: A685 OK LIST completed C: Z432 LIST "" * S: * LIST () "." INBOX S: * LIST () "." INBOX.bar S: Z432 OK LIST completed C: Z433 RENAME INBOX old-mail S: Z433 OK RENAME completed C: Z434 LIST "" * S: * LIST () "." INBOX S: * LIST () "." INBOX.bar S: * LIST () "." old-mail S: Z434 OK LIST completed Crispin Standards Track [Page 38] RFC 3501 IMAPv4 March 2003 6.3.6. SUBSCRIBE Command Arguments: mailbox Responses: no specific responses for this command Result: OK - subscribe completed NO - subscribe failure: can't subscribe to that name BAD - command unknown or arguments invalid The SUBSCRIBE command adds the specified mailbox name to the server's set of "active" or "subscribed" mailboxes as returned by the LSUB command. This command returns a tagged OK response only if the subscription is successful. A server MAY validate the mailbox argument to SUBSCRIBE to verify that it exists. However, it MUST NOT unilaterally remove an existing mailbox name from the subscription list even if a mailbox by that name no longer exists. Note: This requirement is because a server site can choose to routinely remove a mailbox with a well-known name (e.g., "system-alerts") after its contents expire, with the intention of recreating it when new contents are appropriate. Example: C: A002 SUBSCRIBE #news.comp.mail.mime S: A002 OK SUBSCRIBE completed 6.3.7. UNSUBSCRIBE Command Arguments: mailbox name Responses: no specific responses for this command Result: OK - unsubscribe completed NO - unsubscribe failure: can't unsubscribe that name BAD - command unknown or arguments invalid The UNSUBSCRIBE command removes the specified mailbox name from the server's set of "active" or "subscribed" mailboxes as returned by the LSUB command. This command returns a tagged OK response only if the unsubscription is successful. Example: C: A002 UNSUBSCRIBE #news.comp.mail.mime S: A002 OK UNSUBSCRIBE completed Crispin Standards Track [Page 39] RFC 3501 IMAPv4 March 2003 6.3.8. LIST Command Arguments: reference name mailbox name with possible wildcards Responses: untagged responses: LIST Result: OK - list completed NO - list failure: can't list that reference or name BAD - command unknown or arguments invalid The LIST command returns a subset of names from the complete set of all names available to the client. Zero or more untagged LIST replies are returned, containing the name attributes, hierarchy delimiter, and name; see the description of the LIST reply for more detail. The LIST command SHOULD return its data quickly, without undue delay. For example, it SHOULD NOT go to excess trouble to calculate the \Marked or \Unmarked status or perform other processing; if each name requires 1 second of processing, then a list of 1200 names would take 20 minutes! An empty ("" string) reference name argument indicates that the mailbox name is interpreted as by SELECT. The returned mailbox names MUST match the supplied mailbox name pattern. A non-empty reference name argument is the name of a mailbox or a level of mailbox hierarchy, and indicates the context in which the mailbox name is interpreted. An empty ("" string) mailbox name argument is a special request to return the hierarchy delimiter and the root name of the name given in the reference. The value returned as the root MAY be the empty string if the reference is non-rooted or is an empty string. In all cases, a hierarchy delimiter (or NIL if there is no hierarchy) is returned. This permits a client to get the hierarchy delimiter (or find out that the mailbox names are flat) even when no mailboxes by that name currently exist. The reference and mailbox name arguments are interpreted into a canonical form that represents an unambiguous left-to-right hierarchy. The returned mailbox names will be in the interpreted form. Crispin Standards Track [Page 40] RFC 3501 IMAPv4 March 2003 Note: The interpretation of the reference argument is implementation-defined. It depends upon whether the server implementation has a concept of the "current working directory" and leading "break out characters", which override the current working directory. For example, on a server which exports a UNIX or NT filesystem, the reference argument contains the current working directory, and the mailbox name argument would contain the name as interpreted in the current working directory. If a server implementation has no concept of break out characters, the canonical form is normally the reference name appended with the mailbox name. Note that if the server implements the namespace convention (section 5.1.2), "#" is a break out character and must be treated as such. If the reference argument is not a level of mailbox hierarchy (that is, it is a \NoInferiors name), and/or the reference argument does not end with the hierarchy delimiter, it is implementation-dependent how this is interpreted. For example, a reference of "foo/bar" and mailbox name of "rag/baz" could be interpreted as "foo/bar/rag/baz", "foo/barrag/baz", or "foo/rag/baz". A client SHOULD NOT use such a reference argument except at the explicit request of the user. A hierarchical browser MUST NOT make any assumptions about server interpretation of the reference unless the reference is a level of mailbox hierarchy AND ends with the hierarchy delimiter. Any part of the reference argument that is included in the interpreted form SHOULD prefix the interpreted form. It SHOULD also be in the same form as the reference name argument. This rule permits the client to determine if the returned mailbox name is in the context of the reference argument, or if something about the mailbox argument overrode the reference argument. Without this rule, the client would have to have knowledge of the server's naming semantics including what characters are "breakouts" that override a naming context. Crispin Standards Track [Page 41] RFC 3501 IMAPv4 March 2003 For example, here are some examples of how references and mailbox names might be interpreted on a UNIX-based server: Reference Mailbox Name Interpretation ------------ ------------ -------------- ~smith/Mail/ foo.* ~smith/Mail/foo.* archive/ % archive/% #news. comp.mail.* #news.comp.mail.* ~smith/Mail/ /usr/doc/foo /usr/doc/foo archive/ ~fred/Mail/* ~fred/Mail/* The first three examples demonstrate interpretations in the context of the reference argument. Note that "~smith/Mail" SHOULD NOT be transformed into something like "/u2/users/smith/Mail", or it would be impossible for the client to determine that the interpretation was in the context of the reference. The character "*" is a wildcard, and matches zero or more characters at this position. The character "%" is similar to "*", but it does not match a hierarchy delimiter. If the "%" wildcard is the last character of a mailbox name argument, matching levels of hierarchy are also returned. If these levels of hierarchy are not also selectable mailboxes, they are returned with the \Noselect mailbox name attribute (see the description of the LIST response for more details). Server implementations are permitted to "hide" otherwise accessible mailboxes from the wildcard characters, by preventing certain characters or names from matching a wildcard in certain situations. For example, a UNIX-based server might restrict the interpretation of "*" so that an initial "/" character does not match. The special name INBOX is included in the output from LIST, if INBOX is supported by this server for this user and if the uppercase string "INBOX" matches the interpreted reference and mailbox name arguments with wildcards as described above. The criteria for omitting INBOX is whether SELECT INBOX will return failure; it is not relevant whether the user's real INBOX resides on this or some other server. Crispin Standards Track [Page 42] RFC 3501 IMAPv4 March 2003 Example: C: A101 LIST "" "" S: * LIST (\Noselect) "/" "" S: A101 OK LIST Completed C: A102 LIST #news.comp.mail.misc "" S: * LIST (\Noselect) "." #news. S: A102 OK LIST Completed C: A103 LIST /usr/staff/jones "" S: * LIST (\Noselect) "/" / S: A103 OK LIST Completed C: A202 LIST ~/Mail/ % S: * LIST (\Noselect) "/" ~/Mail/foo S: * LIST () "/" ~/Mail/meetings S: A202 OK LIST completed 6.3.9. LSUB Command Arguments: reference name mailbox name with possible wildcards Responses: untagged responses: LSUB Result: OK - lsub completed NO - lsub failure: can't list that reference or name BAD - command unknown or arguments invalid The LSUB command returns a subset of names from the set of names that the user has declared as being "active" or "subscribed". Zero or more untagged LSUB replies are returned. The arguments to LSUB are in the same form as those for LIST. The returned untagged LSUB response MAY contain different mailbox flags from a LIST untagged response. If this should happen, the flags in the untagged LIST are considered more authoritative. A special situation occurs when using LSUB with the % wildcard. Consider what happens if "foo/bar" (with a hierarchy delimiter of "/") is subscribed but "foo" is not. A "%" wildcard to LSUB must return foo, not foo/bar, in the LSUB response, and it MUST be flagged with the \Noselect attribute. The server MUST NOT unilaterally remove an existing mailbox name from the subscription list even if a mailbox by that name no longer exists. Crispin Standards Track [Page 43] RFC 3501 IMAPv4 March 2003 Example: C: A002 LSUB "#news." "comp.mail.*" S: * LSUB () "." #news.comp.mail.mime S: * LSUB () "." #news.comp.mail.misc S: A002 OK LSUB completed C: A003 LSUB "#news." "comp.%" S: * LSUB (\NoSelect) "." #news.comp.mail S: A003 OK LSUB completed 6.3.10. STATUS Command Arguments: mailbox name status data item names Responses: untagged responses: STATUS Result: OK - status completed NO - status failure: no status for that name BAD - command unknown or arguments invalid The STATUS command requests the status of the indicated mailbox. It does not change the currently selected mailbox, nor does it affect the state of any messages in the queried mailbox (in particular, STATUS MUST NOT cause messages to lose the \Recent flag). The STATUS command provides an alternative to opening a second IMAP4rev1 connection and doing an EXAMINE command on a mailbox to query that mailbox's status without deselecting the current mailbox in the first IMAP4rev1 connection. Unlike the LIST command, the STATUS command is not guaranteed to be fast in its response. Under certain circumstances, it can be quite slow. In some implementations, the server is obliged to open the mailbox read-only internally to obtain certain status information. Also unlike the LIST command, the STATUS command does not accept wildcards. Note: The STATUS command is intended to access the status of mailboxes other than the currently selected mailbox. Because the STATUS command can cause the mailbox to be opened internally, and because this information is available by other means on the selected mailbox, the STATUS command SHOULD NOT be used on the currently selected mailbox. Crispin Standards Track [Page 44] RFC 3501 IMAPv4 March 2003 The STATUS command MUST NOT be used as a "check for new messages in the selected mailbox" operation (refer to sections 7, 7.3.1, and 7.3.2 for more information about the proper method for new message checking). Because the STATUS command is not guaranteed to be fast in its results, clients SHOULD NOT expect to be able to issue many consecutive STATUS commands and obtain reasonable performance. The currently defined status data items that can be requested are: MESSAGES The number of messages in the mailbox. RECENT The number of messages with the \Recent flag set. UIDNEXT The next unique identifier value of the mailbox. Refer to section 2.3.1.1 for more information. UIDVALIDITY The unique identifier validity value of the mailbox. Refer to section 2.3.1.1 for more information. UNSEEN The number of messages which do not have the \Seen flag set. Example: C: A042 STATUS blurdybloop (UIDNEXT MESSAGES) S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) S: A042 OK STATUS completed Crispin Standards Track [Page 45] RFC 3501 IMAPv4 March 2003 6.3.11. APPEND Command Arguments: mailbox name OPTIONAL flag parenthesized list OPTIONAL date/time string message literal Responses: no specific responses for this command Result: OK - append completed NO - append error: can't append to that mailbox, error in flags or date/time or message text BAD - command unknown or arguments invalid The APPEND command appends the literal argument as a new message to the end of the specified destination mailbox. This argument SHOULD be in the format of an [RFC-2822] message. 8-bit characters are permitted in the message. A server implementation that is unable to preserve 8-bit data properly MUST be able to reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] content transfer encoding. Note: There MAY be exceptions, e.g., draft messages, in which required [RFC-2822] header lines are omitted in the message literal argument to APPEND. The full implications of doing so MUST be understood and carefully weighed. If a flag parenthesized list is specified, the flags SHOULD be set in the resulting message; otherwise, the flag list of the resulting message is set to empty by default. In either case, the Recent flag is also set. If a date-time is specified, the internal date SHOULD be set in the resulting message; otherwise, the internal date of the resulting message is set to the current date and time by default. If the append is unsuccessful for any reason, the mailbox MUST be restored to its state before the APPEND attempt; no partial appending is permitted. If the destination mailbox does not exist, a server MUST return an error, and MUST NOT automatically create the mailbox. Unless it is certain that the destination mailbox can not be created, the server MUST send the response code "[TRYCREATE]" as the prefix of the text of the tagged NO response. This gives a hint to the client that it can attempt a CREATE command and retry the APPEND if the CREATE is successful. Crispin Standards Track [Page 46] RFC 3501 IMAPv4 March 2003 If the mailbox is currently selected, the normal new message actions SHOULD occur. Specifically, the server SHOULD notify the client immediately via an untagged EXISTS response. If the server does not do so, the client MAY issue a NOOP command (or failing that, a CHECK command) after one or more APPEND commands. Example: C: A003 APPEND saved-messages (\Seen) {310} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK APPEND completed Note: The APPEND command is not used for message delivery, because it does not provide a mechanism to transfer [SMTP] envelope information. 6.4. Client Commands - Selected State In the selected state, commands that manipulate messages in a mailbox are permitted. In addition to the universal commands (CAPABILITY, NOOP, and LOGOUT), and the authenticated state commands (SELECT, EXAMINE, CREATE, DELETE, RENAME, SUBSCRIBE, UNSUBSCRIBE, LIST, LSUB, STATUS, and APPEND), the following commands are valid in the selected state: CHECK, CLOSE, EXPUNGE, SEARCH, FETCH, STORE, COPY, and UID. 6.4.1. CHECK Command Arguments: none Responses: no specific responses for this command Result: OK - check completed BAD - command unknown or arguments invalid The CHECK command requests a checkpoint of the currently selected mailbox. A checkpoint refers to any implementation-dependent housekeeping associated with the mailbox (e.g., resolving the server's in-memory state of the mailbox with the state on its Crispin Standards Track [Page 47] RFC 3501 IMAPv4 March 2003 disk) that is not normally executed as part of each command. A checkpoint MAY take a non-instantaneous amount of real time to complete. If a server implementation has no such housekeeping considerations, CHECK is equivalent to NOOP. There is no guarantee that an EXISTS untagged response will happen as a result of CHECK. NOOP, not CHECK, SHOULD be used for new message polling. Example: C: FXXZ CHECK S: FXXZ OK CHECK Completed 6.4.2. CLOSE Command Arguments: none Responses: no specific responses for this command Result: OK - close completed, now in authenticated state BAD - command unknown or arguments invalid The CLOSE command permanently removes all messages that have the \Deleted flag set from the currently selected mailbox, and returns to the authenticated state from the selected state. No untagged EXPUNGE responses are sent. No messages are removed, and no error is given, if the mailbox is selected by an EXAMINE command or is otherwise selected read-only. Even if a mailbox is selected, a SELECT, EXAMINE, or LOGOUT command MAY be issued without previously issuing a CLOSE command. The SELECT, EXAMINE, and LOGOUT commands implicitly close the currently selected mailbox without doing an expunge. However, when many messages are deleted, a CLOSE-LOGOUT or CLOSE-SELECT sequence is considerably faster than an EXPUNGE-LOGOUT or EXPUNGE-SELECT because no untagged EXPUNGE responses (which the client would probably ignore) are sent. Example: C: A341 CLOSE S: A341 OK CLOSE completed Crispin Standards Track [Page 48] RFC 3501 IMAPv4 March 2003 6.4.3. EXPUNGE Command Arguments: none Responses: untagged responses: EXPUNGE Result: OK - expunge completed NO - expunge failure: can't expunge (e.g., permission denied) BAD - command unknown or arguments invalid The EXPUNGE command permanently removes all messages that have the \Deleted flag set from the currently selected mailbox. Before returning an OK to the client, an untagged EXPUNGE response is sent for each message that is removed. Example: C: A202 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 5 EXPUNGE S: * 8 EXPUNGE S: A202 OK EXPUNGE completed Note: In this example, messages 3, 4, 7, and 11 had the \Deleted flag set. See the description of the EXPUNGE response for further explanation. 6.4.4. SEARCH Command Arguments: OPTIONAL [CHARSET] specification searching criteria (one or more) Responses: REQUIRED untagged response: SEARCH Result: OK - search completed NO - search error: can't search that [CHARSET] or criteria BAD - command unknown or arguments invalid The SEARCH command searches the mailbox for messages that match the given searching criteria. Searching criteria consist of one or more search keys. The untagged SEARCH response from the server contains a listing of message sequence numbers corresponding to those messages that match the searching criteria. Crispin Standards Track [Page 49] RFC 3501 IMAPv4 March 2003 When multiple keys are specified, the result is the intersection (AND function) of all the messages that match those keys. For example, the criteria DELETED FROM "SMITH" SINCE 1-Feb-1994 refers to all deleted messages from Smith that were placed in the mailbox since February 1, 1994. A search key can also be a parenthesized list of one or more search keys (e.g., for use with the OR and NOT keys). Server implementations MAY exclude [MIME-IMB] body parts with terminal content media types other than TEXT and MESSAGE from consideration in SEARCH matching. The OPTIONAL [CHARSET] specification consists of the word "CHARSET" followed by a registered [CHARSET]. It indicates the [CHARSET] of the strings that appear in the search criteria. [MIME-IMB] content transfer encodings, and [MIME-HDRS] strings in [RFC-2822]/[MIME-IMB] headers, MUST be decoded before comparing text in a [CHARSET] other than US-ASCII. US-ASCII MUST be supported; other [CHARSET]s MAY be supported. If the server does not support the specified [CHARSET], it MUST return a tagged NO response (not a BAD). This response SHOULD contain the BADCHARSET response code, which MAY list the [CHARSET]s supported by the server. In all search keys that use strings, a message matches the key if the string is a substring of the field. The matching is case-insensitive. The defined search keys are as follows. Refer to the Formal Syntax section for the precise syntactic definitions of the arguments. Messages with message sequence numbers corresponding to the specified message sequence number set. ALL All messages in the mailbox; the default initial key for ANDing. ANSWERED Messages with the \Answered flag set. Crispin Standards Track [Page 50] RFC 3501 IMAPv4 March 2003 BCC Messages that contain the specified string in the envelope structure's BCC field. BEFORE Messages whose internal date (disregarding time and timezone) is earlier than the specified date. BODY Messages that contain the specified string in the body of the message. CC Messages that contain the specified string in the envelope structure's CC field. DELETED Messages with the \Deleted flag set. DRAFT Messages with the \Draft flag set. FLAGGED Messages with the \Flagged flag set. FROM Messages that contain the specified string in the envelope structure's FROM field. HEADER Messages that have a header with the specified field-name (as defined in [RFC-2822]) and that contains the specified string in the text of the header (what comes after the colon). If the string to search is zero-length, this matches all messages that have a header line with the specified field-name regardless of the contents. KEYWORD Messages with the specified keyword flag set. LARGER Messages with an [RFC-2822] size larger than the specified number of octets. NEW Messages that have the \Recent flag set but not the \Seen flag. This is functionally equivalent to "(RECENT UNSEEN)". Crispin Standards Track [Page 51] RFC 3501 IMAPv4 March 2003 NOT Messages that do not match the specified search key. OLD Messages that do not have the \Recent flag set. This is functionally equivalent to "NOT RECENT" (as opposed to "NOT NEW"). ON Messages whose internal date (disregarding time and timezone) is within the specified date. OR Messages that match either search key. RECENT Messages that have the \Recent flag set. SEEN Messages that have the \Seen flag set. SENTBEFORE Messages whose [RFC-2822] Date: header (disregarding time and timezone) is earlier than the specified date. SENTON Messages whose [RFC-2822] Date: header (disregarding time and timezone) is within the specified date. SENTSINCE Messages whose [RFC-2822] Date: header (disregarding time and timezone) is within or later than the specified date. SINCE Messages whose internal date (disregarding time and timezone) is within or later than the specified date. SMALLER Messages with an [RFC-2822] size smaller than the specified number of octets. Crispin Standards Track [Page 52] RFC 3501 IMAPv4 March 2003 SUBJECT Messages that contain the specified string in the envelope structure's SUBJECT field. TEXT Messages that contain the specified string in the header or body of the message. TO Messages that contain the specified string in the envelope structure's TO field. UID Messages with unique identifiers corresponding to the specified unique identifier set. Sequence set ranges are permitted. UNANSWERED Messages that do not have the \Answered flag set. UNDELETED Messages that do not have the \Deleted flag set. UNDRAFT Messages that do not have the \Draft flag set. UNFLAGGED Messages that do not have the \Flagged flag set. UNKEYWORD Messages that do not have the specified keyword flag set. UNSEEN Messages that do not have the \Seen flag set. Crispin Standards Track [Page 53] RFC 3501 IMAPv4 March 2003 Example: C: A282 SEARCH FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" S: * SEARCH 2 84 882 S: A282 OK SEARCH completed C: A283 SEARCH TEXT "string not in mailbox" S: * SEARCH S: A283 OK SEARCH completed C: A284 SEARCH CHARSET UTF-8 TEXT {6} C: XXXXXX S: * SEARCH 43 S: A284 OK SEARCH completed Note: Since this document is restricted to 7-bit ASCII text, it is not possible to show actual UTF-8 data. The "XXXXXX" is a placeholder for what would be 6 octets of 8-bit data in an actual transaction. 6.4.5. FETCH Command Arguments: sequence set message data item names or macro Responses: untagged responses: FETCH Result: OK - fetch completed NO - fetch error: can't fetch that data BAD - command unknown or arguments invalid The FETCH command retrieves data associated with a message in the mailbox. The data items to be fetched can be either a single atom or a parenthesized list. Most data items, identified in the formal syntax under the msg-att-static rule, are static and MUST NOT change for any particular message. Other data items, identified in the formal syntax under the msg-att-dynamic rule, MAY change, either as a result of a STORE command or due to external events. For example, if a client receives an ENVELOPE for a message when it already knows the envelope, it can safely ignore the newly transmitted envelope. There are three macros which specify commonly-used sets of data items, and can be used instead of data items. A macro must be used by itself, and not in conjunction with other macros or data items. Crispin Standards Track [Page 54] RFC 3501 IMAPv4 March 2003 ALL Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE) FAST Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE) FULL Macro equivalent to: (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY) The currently defined data items that can be fetched are: BODY Non-extensible form of BODYSTRUCTURE. BODY[
]<> The text of a particular body section. The section specification is a set of zero or more part specifiers delimited by periods. A part specifier is either a part number or one of the following: HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, MIME, and TEXT. An empty section specification refers to the entire message, including the header. Every message has at least one part number. Non-[MIME-IMB] messages, and non-multipart [MIME-IMB] messages with no encapsulated message, only have a part 1. Multipart messages are assigned consecutive part numbers, as they occur in the message. If a particular part is of type message or multipart, its parts MUST be indicated by a period followed by the part number within that nested multipart part. A part of type MESSAGE/RFC822 also has nested part numbers, referring to parts of the MESSAGE part's body. The HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, and TEXT part specifiers can be the sole part specifier or can be prefixed by one or more numeric part specifiers, provided that the numeric part specifier refers to a part of type MESSAGE/RFC822. The MIME part specifier MUST be prefixed by one or more numeric part specifiers. The HEADER, HEADER.FIELDS, and HEADER.FIELDS.NOT part specifiers refer to the [RFC-2822] header of the message or of an encapsulated [MIME-IMT] MESSAGE/RFC822 message. HEADER.FIELDS and HEADER.FIELDS.NOT are followed by a list of field-name (as defined in [RFC-2822]) names, and return a Crispin Standards Track [Page 55] RFC 3501 IMAPv4 March 2003 subset of the header. The subset returned by HEADER.FIELDS contains only those header fields with a field-name that matches one of the names in the list; similarly, the subset returned by HEADER.FIELDS.NOT contains only the header fields with a non-matching field-name. The field-matching is case-insensitive but otherwise exact. Subsetting does not exclude the [RFC-2822] delimiting blank line between the header and the body; the blank line is included in all header fetches, except in the case of a message which has no body and no blank line. The MIME part specifier refers to the [MIME-IMB] header for this part. The TEXT part specifier refers to the text body of the message, omitting the [RFC-2822] header. Here is an example of a complex message with some of its part specifiers: HEADER ([RFC-2822] header of the message) TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED 1 TEXT/PLAIN 2 APPLICATION/OCTET-STREAM 3 MESSAGE/RFC822 3.HEADER ([RFC-2822] header of the message) 3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED 3.1 TEXT/PLAIN 3.2 APPLICATION/OCTET-STREAM 4 MULTIPART/MIXED 4.1 IMAGE/GIF 4.1.MIME ([MIME-IMB] header for the IMAGE/GIF) 4.2 MESSAGE/RFC822 4.2.HEADER ([RFC-2822] header of the message) 4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED 4.2.1 TEXT/PLAIN 4.2.2 MULTIPART/ALTERNATIVE 4.2.2.1 TEXT/PLAIN 4.2.2.2 TEXT/RICHTEXT It is possible to fetch a substring of the designated text. This is done by appending an open angle bracket ("<"), the octet position of the first desired octet, a period, the maximum number of octets desired, and a close angle bracket (">") to the part specifier. If the starting octet is beyond the end of the text, an empty string is returned. Crispin Standards Track [Page 56] RFC 3501 IMAPv4 March 2003 Any partial fetch that attempts to read beyond the end of the text is truncated as appropriate. A partial fetch that starts at octet 0 is returned as a partial fetch, even if this truncation happened. Note: This means that BODY[]<0.2048> of a 1500-octet message will return BODY[]<0> with a literal of size 1500, not BODY[]. Note: A substring fetch of a HEADER.FIELDS or HEADER.FIELDS.NOT part specifier is calculated after subsetting the header. The \Seen flag is implicitly set; if this causes the flags to change, they SHOULD be included as part of the FETCH responses. BODY.PEEK[
]<> An alternate form of BODY[
] that does not implicitly set the \Seen flag. BODYSTRUCTURE The [MIME-IMB] body structure of the message. This is computed by the server by parsing the [MIME-IMB] header fields in the [RFC-2822] header and [MIME-IMB] headers. ENVELOPE The envelope structure of the message. This is computed by the server by parsing the [RFC-2822] header into the component parts, defaulting various fields as necessary. FLAGS The flags that are set for this message. INTERNALDATE The internal date of the message. RFC822 Functionally equivalent to BODY[], differing in the syntax of the resulting untagged FETCH data (RFC822 is returned). RFC822.HEADER Functionally equivalent to BODY.PEEK[HEADER], differing in the syntax of the resulting untagged FETCH data (RFC822.HEADER is returned). RFC822.SIZE The [RFC-2822] size of the message. Crispin Standards Track [Page 57] RFC 3501 IMAPv4 March 2003 RFC822.TEXT Functionally equivalent to BODY[TEXT], differing in the syntax of the resulting untagged FETCH data (RFC822.TEXT is returned). UID The unique identifier for the message. Example: C: A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) S: * 2 FETCH .... S: * 3 FETCH .... S: * 4 FETCH .... S: A654 OK FETCH completed 6.4.6. STORE Command Arguments: sequence set message data item name value for message data item Responses: untagged responses: FETCH Result: OK - store completed NO - store error: can't store that data BAD - command unknown or arguments invalid The STORE command alters data associated with a message in the mailbox. Normally, STORE will return the updated value of the data with an untagged FETCH response. A suffix of ".SILENT" in the data item name prevents the untagged FETCH, and the server SHOULD assume that the client has determined the updated value itself or does not care about the updated value. Note: Regardless of whether or not the ".SILENT" suffix was used, the server SHOULD send an untagged FETCH response if a change to a message's flags from an external source is observed. The intent is that the status of the flags is determinate without a race condition. Crispin Standards Track [Page 58] RFC 3501 IMAPv4 March 2003 The currently defined data items that can be stored are: FLAGS Replace the flags for the message (other than \Recent) with the argument. The new value of the flags is returned as if a FETCH of those flags was done. FLAGS.SILENT Equivalent to FLAGS, but without returning a new value. +FLAGS Add the argument to the flags for the message. The new value of the flags is returned as if a FETCH of those flags was done. +FLAGS.SILENT Equivalent to +FLAGS, but without returning a new value. -FLAGS Remove the argument from the flags for the message. The new value of the flags is returned as if a FETCH of those flags was done. -FLAGS.SILENT Equivalent to -FLAGS, but without returning a new value. Example: C: A003 STORE 2:4 +FLAGS (\Deleted) S: * 2 FETCH (FLAGS (\Deleted \Seen)) S: * 3 FETCH (FLAGS (\Deleted)) S: * 4 FETCH (FLAGS (\Deleted \Flagged \Seen)) S: A003 OK STORE completed 6.4.7. COPY Command Arguments: sequence set mailbox name Responses: no specific responses for this command Result: OK - copy completed NO - copy error: can't copy those messages or to that name BAD - command unknown or arguments invalid Crispin Standards Track [Page 59] RFC 3501 IMAPv4 March 2003 The COPY command copies the specified message(s) to the end of the specified destination mailbox. The flags and internal date of the message(s) SHOULD be preserved, and the Recent flag SHOULD be set, in the copy. If the destination mailbox does not exist, a server SHOULD return an error. It SHOULD NOT automatically create the mailbox. Unless it is certain that the destination mailbox can not be created, the server MUST send the response code "[TRYCREATE]" as the prefix of the text of the tagged NO response. This gives a hint to the client that it can attempt a CREATE command and retry the COPY if the CREATE is successful. If the COPY command is unsuccessful for any reason, server implementations MUST restore the destination mailbox to its state before the COPY attempt. Example: C: A003 COPY 2:4 MEETING S: A003 OK COPY completed 6.4.8. UID Command Arguments: command name command arguments Responses: untagged responses: FETCH, SEARCH Result: OK - UID command completed NO - UID command error BAD - command unknown or arguments invalid The UID command has two forms. In the first form, it takes as its arguments a COPY, FETCH, or STORE command with arguments appropriate for the associated command. However, the numbers in the sequence set argument are unique identifiers instead of message sequence numbers. Sequence set ranges are permitted, but there is no guarantee that unique identifiers will be contiguous. A non-existent unique identifier is ignored without any error message generated. Thus, it is possible for a UID FETCH command to return an OK without any data or a UID COPY or UID STORE to return an OK without performing any operations. In the second form, the UID command takes a SEARCH command with SEARCH command arguments. The interpretation of the arguments is the same as with SEARCH; however, the numbers returned in a SEARCH response for a UID SEARCH command are unique identifiers instead Crispin Standards Track [Page 60] RFC 3501 IMAPv4 March 2003 of message sequence numbers. For example, the command UID SEARCH 1:100 UID 443:557 returns the unique identifiers corresponding to the intersection of two sequence sets, the message sequence number range 1:100 and the UID range 443:557. Note: in the above example, the UID range 443:557 appears. The same comment about a non-existent unique identifier being ignored without any error message also applies here. Hence, even if neither UID 443 or 557 exist, this range is valid and would include an existing UID 495. Also note that a UID range of 559:* always includes the UID of the last message in the mailbox, even if 559 is higher than any assigned UID value. This is because the contents of a range are independent of the order of the range endpoints. Thus, any UID range with * as one of the endpoints indicates at least one message (the message with the highest numbered UID), unless the mailbox is empty. The number after the "*" in an untagged FETCH response is always a message sequence number, not a unique identifier, even for a UID command response. However, server implementations MUST implicitly include the UID message data item as part of any FETCH response caused by a UID command, regardless of whether a UID was specified as a message data item to the FETCH. Note: The rule about including the UID message data item as part of a FETCH response primarily applies to the UID FETCH and UID STORE commands, including a UID FETCH command that does not include UID as a message data item. Although it is unlikely that the other UID commands will cause an untagged FETCH, this rule applies to these commands as well. Example: C: A999 UID FETCH 4827313:4828442 FLAGS S: * 23 FETCH (FLAGS (\Seen) UID 4827313) S: * 24 FETCH (FLAGS (\Seen) UID 4827943) S: * 25 FETCH (FLAGS (\Seen) UID 4828442) S: A999 OK UID FETCH completed Crispin Standards Track [Page 61] RFC 3501 IMAPv4 March 2003 6.5. Client Commands - Experimental/Expansion 6.5.1. X Command Arguments: implementation defined Responses: implementation defined Result: OK - command completed NO - failure BAD - command unknown or arguments invalid Any command prefixed with an X is an experimental command. Commands which are not part of this specification, a standard or standards-track revision of this specification, or an IESG-approved experimental protocol, MUST use the X prefix. Any added untagged responses issued by an experimental command MUST also be prefixed with an X. Server implementations MUST NOT send any such untagged responses, unless the client requested it by issuing the associated experimental command. Example: C: a441 CAPABILITY S: * CAPABILITY IMAP4rev1 XPIG-LATIN S: a441 OK CAPABILITY completed C: A442 XPIG-LATIN S: * XPIG-LATIN ow-nay eaking-spay ig-pay atin-lay S: A442 OK XPIG-LATIN ompleted-cay 7. Server Responses Server responses are in three forms: status responses, server data, and command continuation request. The information contained in a server response, identified by "Contents:" in the response descriptions below, is described by function, not by syntax. The precise syntax of server responses is described in the Formal Syntax section. The client MUST be prepared to accept any response at all times. Status responses can be tagged or untagged. Tagged status responses indicate the completion result (OK, NO, or BAD status) of a client command, and have a tag matching the command. Some status responses, and all server data, are untagged. An untagged response is indicated by the token "*" instead of a tag. Untagged status responses indicate server greeting, or server status Crispin Standards Track [Page 62] RFC 3501 IMAPv4 March 2003 that does not indicate the completion of a command (for example, an impending system shutdown alert). For historical reasons, untagged server data responses are also called "unsolicited data", although strictly speaking, only unilateral server data is truly "unsolicited". Certain server data MUST be recorded by the client when it is received; this is noted in the description of that data. Such data conveys critical information which affects the interpretation of all subsequent commands and responses (e.g., updates reflecting the creation or destruction of messages). Other server data SHOULD be recorded for later reference; if the client does not need to record the data, or if recording the data has no obvious purpose (e.g., a SEARCH response when no SEARCH command is in progress), the data SHOULD be ignored. An example of unilateral untagged server data occurs when the IMAP connection is in the selected state. In the selected state, the server checks the mailbox for new messages as part of command execution. Normally, this is part of the execution of every command; hence, a NOOP command suffices to check for new messages. If new messages are found, the server sends untagged EXISTS and RECENT responses reflecting the new size of the mailbox. Server implementations that offer multiple simultaneous access to the same mailbox SHOULD also send appropriate unilateral untagged FETCH and EXPUNGE responses if another agent changes the state of any message flags or expunges any messages. Command continuation request responses use the token "+" instead of a tag. These responses are sent by the server to indicate acceptance of an incomplete client command and readiness for the remainder of the command. 7.1. Server Responses - Status Responses Status responses are OK, NO, BAD, PREAUTH and BYE. OK, NO, and BAD can be tagged or untagged. PREAUTH and BYE are always untagged. Status responses MAY include an OPTIONAL "response code". A response code consists of data inside square brackets in the form of an atom, possibly followed by a space and arguments. The response code contains additional information or status codes for client software beyond the OK/NO/BAD condition, and are defined when there is a specific action that a client can take based upon the additional information. Crispin Standards Track [Page 63] RFC 3501 IMAPv4 March 2003 The currently defined response codes are: ALERT The human-readable text contains a special alert that MUST be presented to the user in a fashion that calls the user's attention to the message. BADCHARSET Optionally followed by a parenthesized list of charsets. A SEARCH failed because the given charset is not supported by this implementation. If the optional list of charsets is given, this lists the charsets that are supported by this implementation. CAPABILITY Followed by a list of capabilities. This can appear in the initial OK or PREAUTH response to transmit an initial capabilities list. This makes it unnecessary for a client to send a separate CAPABILITY command if it recognizes this response. PARSE The human-readable text represents an error in parsing the [RFC-2822] header or [MIME-IMB] headers of a message in the mailbox. PERMANENTFLAGS Followed by a parenthesized list of flags, indicates which of the known flags the client can change permanently. Any flags that are in the FLAGS untagged response, but not the PERMANENTFLAGS list, can not be set permanently. If the client attempts to STORE a flag that is not in the PERMANENTFLAGS list, the server will either ignore the change or store the state change for the remainder of the current session only. The PERMANENTFLAGS list can also include the special flag \*, which indicates that it is possible to create new keywords by attempting to store those flags in the mailbox. Crispin Standards Track [Page 64] RFC 3501 IMAPv4 March 2003 READ-ONLY The mailbox is selected read-only, or its access while selected has changed from read-write to read-only. READ-WRITE The mailbox is selected read-write, or its access while selected has changed from read-only to read-write. TRYCREATE An APPEND or COPY attempt is failing because the target mailbox does not exist (as opposed to some other reason). This is a hint to the client that the operation can succeed if the mailbox is first created by the CREATE command. UIDNEXT Followed by a decimal number, indicates the next unique identifier value. Refer to section 2.3.1.1 for more information. UIDVALIDITY Followed by a decimal number, indicates the unique identifier validity value. Refer to section 2.3.1.1 for more information. UNSEEN Followed by a decimal number, indicates the number of the first message without the \Seen flag set. Additional response codes defined by particular client or server implementations SHOULD be prefixed with an "X" until they are added to a revision of this protocol. Client implementations SHOULD ignore response codes that they do not recognize. 7.1.1. OK Response Contents: OPTIONAL response code human-readable text The OK response indicates an information message from the server. When tagged, it indicates successful completion of the associated command. The human-readable text MAY be presented to the user as an information message. The untagged form indicates an Crispin Standards Track [Page 65] RFC 3501 IMAPv4 March 2003 information-only message; the nature of the information MAY be indicated by a response code. The untagged form is also used as one of three possible greetings at connection startup. It indicates that the connection is not yet authenticated and that a LOGIN command is needed. Example: S: * OK IMAP4rev1 server ready C: A001 LOGIN fred blurdybloop S: * OK [ALERT] System shutdown in 10 minutes S: A001 OK LOGIN Completed 7.1.2. NO Response Contents: OPTIONAL response code human-readable text The NO response indicates an operational error message from the server. When tagged, it indicates unsuccessful completion of the associated command. The untagged form indicates a warning; the command can still complete successfully. The human-readable text describes the condition. Example: C: A222 COPY 1:2 owatagusiam S: * NO Disk is 98% full, please delete unnecessary data S: A222 OK COPY completed C: A223 COPY 3:200 blurdybloop S: * NO Disk is 98% full, please delete unnecessary data S: * NO Disk is 99% full, please delete unnecessary data S: A223 NO COPY failed: disk is full 7.1.3. BAD Response Contents: OPTIONAL response code human-readable text The BAD response indicates an error message from the server. When tagged, it reports a protocol-level error in the client's command; the tag indicates the command that caused the error. The untagged form indicates a protocol-level error for which the associated command can not be determined; it can also indicate an internal server failure. The human-readable text describes the condition. Crispin Standards Track [Page 66] RFC 3501 IMAPv4 March 2003 Example: C: ...very long command line... S: * BAD Command line too long C: ...empty line... S: * BAD Empty command line C: A443 EXPUNGE S: * BAD Disk crash, attempting salvage to a new disk! S: * OK Salvage successful, no data lost S: A443 OK Expunge completed 7.1.4. PREAUTH Response Contents: OPTIONAL response code human-readable text The PREAUTH response is always untagged, and is one of three possible greetings at connection startup. It indicates that the connection has already been authenticated by external means; thus no LOGIN command is needed. Example: S: * PREAUTH IMAP4rev1 server logged in as Smith 7.1.5. BYE Response Contents: OPTIONAL response code human-readable text The BYE response is always untagged, and indicates that the server is about to close the connection. The human-readable text MAY be displayed to the user in a status report by the client. The BYE response is sent under one of four conditions: 1) as part of a normal logout sequence. The server will close the connection after sending the tagged OK response to the LOGOUT command. 2) as a panic shutdown announcement. The server closes the connection immediately. 3) as an announcement of an inactivity autologout. The server closes the connection immediately. 4) as one of three possible greetings at connection startup, indicating that the server is not willing to accept a connection from this client. The server closes the connection immediately. Crispin Standards Track [Page 67] RFC 3501 IMAPv4 March 2003 The difference between a BYE that occurs as part of a normal LOGOUT sequence (the first case) and a BYE that occurs because of a failure (the other three cases) is that the connection closes immediately in the failure case. In all cases the client SHOULD continue to read response data from the server until the connection is closed; this will ensure that any pending untagged or completion responses are read and processed. Example: S: * BYE Autologout; idle for too long 7.2. Server Responses - Server and Mailbox Status These responses are always untagged. This is how server and mailbox status data are transmitted from the server to the client. Many of these responses typically result from a command with the same name. 7.2.1. CAPABILITY Response Contents: capability listing The CAPABILITY response occurs as a result of a CAPABILITY command. The capability listing contains a space-separated listing of capability names that the server supports. The capability listing MUST include the atom "IMAP4rev1". In addition, client and server implementations MUST implement the STARTTLS, LOGINDISABLED, and AUTH=PLAIN (described in [IMAP-TLS]) capabilities. See the Security Considerations section for important information. A capability name which begins with "AUTH=" indicates that the server supports that particular authentication mechanism. The LOGINDISABLED capability indicates that the LOGIN command is disabled, and that the server will respond with a tagged NO response to any attempt to use the LOGIN command even if the user name and password are valid. An IMAP client MUST NOT issue the LOGIN command if the server advertises the LOGINDISABLED capability. Other capability names indicate that the server supports an extension, revision, or amendment to the IMAP4rev1 protocol. Server responses MUST conform to this document until the client issues a command that uses the associated capability. Capability names MUST either begin with "X" or be standard or standards-track IMAP4rev1 extensions, revisions, or amendments registered with IANA. A server MUST NOT offer unregistered or Crispin Standards Track [Page 68] RFC 3501 IMAPv4 March 2003 non-standard capability names, unless such names are prefixed with an "X". Client implementations SHOULD NOT require any capability name other than "IMAP4rev1", and MUST ignore any unknown capability names. A server MAY send capabilities automatically, by using the CAPABILITY response code in the initial PREAUTH or OK responses, and by sending an updated CAPABILITY response code in the tagged OK response as part of a successful authentication. It is unnecessary for a client to send a separate CAPABILITY command if it recognizes these automatic capabilities. Example: S: * CAPABILITY IMAP4rev1 STARTTLS AUTH=GSSAPI XPIG-LATIN 7.2.2. LIST Response Contents: name attributes hierarchy delimiter name The LIST response occurs as a result of a LIST command. It returns a single name that matches the LIST specification. There can be multiple LIST responses for a single LIST command. Four name attributes are defined: \Noinferiors It is not possible for any child levels of hierarchy to exist under this name; no child levels exist now and none can be created in the future. \Noselect It is not possible to use this name as a selectable mailbox. \Marked The mailbox has been marked "interesting" by the server; the mailbox probably contains messages that have been added since the last time the mailbox was selected. \Unmarked The mailbox does not contain any additional messages since the last time the mailbox was selected. Crispin Standards Track [Page 69] RFC 3501 IMAPv4 March 2003 If it is not feasible for the server to determine whether or not the mailbox is "interesting", or if the name is a \Noselect name, the server SHOULD NOT send either \Marked or \Unmarked. The hierarchy delimiter is a character used to delimit levels of hierarchy in a mailbox name. A client can use it to create child mailboxes, and to search higher or lower levels of naming hierarchy. All children of a top-level hierarchy node MUST use the same separator character. A NIL hierarchy delimiter means that no hierarchy exists; the name is a "flat" name. The name represents an unambiguous left-to-right hierarchy, and MUST be valid for use as a reference in LIST and LSUB commands. Unless \Noselect is indicated, the name MUST also be valid as an argument for commands, such as SELECT, that accept mailbox names. Example: S: * LIST (\Noselect) "/" ~/Mail/foo 7.2.3. LSUB Response Contents: name attributes hierarchy delimiter name The LSUB response occurs as a result of an LSUB command. It returns a single name that matches the LSUB specification. There can be multiple LSUB responses for a single LSUB command. The data is identical in format to the LIST response. Example: S: * LSUB () "." #news.comp.mail.misc 7.2.4 STATUS Response Contents: name status parenthesized list The STATUS response occurs as a result of an STATUS command. It returns the mailbox name that matches the STATUS specification and the requested mailbox status information. Example: S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292) Crispin Standards Track [Page 70] RFC 3501 IMAPv4 March 2003 7.2.5. SEARCH Response Contents: zero or more numbers The SEARCH response occurs as a result of a SEARCH or UID SEARCH command. The number(s) refer to those messages that match the search criteria. For SEARCH, these are message sequence numbers; for UID SEARCH, these are unique identifiers. Each number is delimited by a space. Example: S: * SEARCH 2 3 6 7.2.6. FLAGS Response Contents: flag parenthesized list The FLAGS response occurs as a result of a SELECT or EXAMINE command. The flag parenthesized list identifies the flags (at a minimum, the system-defined flags) that are applicable for this mailbox. Flags other than the system flags can also exist, depending on server implementation. The update from the FLAGS response MUST be recorded by the client. Example: S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) 7.3. Server Responses - Mailbox Size These responses are always untagged. This is how changes in the size of the mailbox are transmitted from the server to the client. Immediately following the "*" token is a number that represents a message count. 7.3.1. EXISTS Response Contents: none The EXISTS response reports the number of messages in the mailbox. This response occurs as a result of a SELECT or EXAMINE command, and if the size of the mailbox changes (e.g., new messages). The update from the EXISTS response MUST be recorded by the client. Example: S: * 23 EXISTS Crispin Standards Track [Page 71] RFC 3501 IMAPv4 March 2003 7.3.2. RECENT Response Contents: none The RECENT response reports the number of messages with the \Recent flag set. This response occurs as a result of a SELECT or EXAMINE command, and if the size of the mailbox changes (e.g., new messages). Note: It is not guaranteed that the message sequence numbers of recent messages will be a contiguous range of the highest n messages in the mailbox (where n is the value reported by the RECENT response). Examples of situations in which this is not the case are: multiple clients having the same mailbox open (the first session to be notified will see it as recent, others will probably see it as non-recent), and when the mailbox is re-ordered by a non-IMAP agent. The only reliable way to identify recent messages is to look at message flags to see which have the \Recent flag set, or to do a SEARCH RECENT. The update from the RECENT response MUST be recorded by the client. Example: S: * 5 RECENT 7.4. Server Responses - Message Status These responses are always untagged. This is how message data are transmitted from the server to the client, often as a result of a command with the same name. Immediately following the "*" token is a number that represents a message sequence number. 7.4.1. EXPUNGE Response Contents: none The EXPUNGE response reports that the specified message sequence number has been permanently removed from the mailbox. The message sequence number for each successive message in the mailbox is immediately decremented by 1, and this decrement is reflected in message sequence numbers in subsequent responses (including other untagged EXPUNGE responses). Crispin Standards Track [Page 72] RFC 3501 IMAPv4 March 2003 The EXPUNGE response also decrements the number of messages in the mailbox; it is not necessary to send an EXISTS response with the new value. As a result of the immediate decrement rule, message sequence numbers that appear in a set of successive EXPUNGE responses depend upon whether the messages are removed starting from lower numbers to higher numbers, or from higher numbers to lower numbers. For example, if the last 5 messages in a 9-message mailbox are expunged, a "lower to higher" server will send five untagged EXPUNGE responses for message sequence number 5, whereas a "higher to lower server" will send successive untagged EXPUNGE responses for message sequence numbers 9, 8, 7, 6, and 5. An EXPUNGE response MUST NOT be sent when no command is in progress, nor while responding to a FETCH, STORE, or SEARCH command. This rule is necessary to prevent a loss of synchronization of message sequence numbers between client and server. A command is not "in progress" until the complete command has been received; in particular, a command is not "in progress" during the negotiation of command continuation. Note: UID FETCH, UID STORE, and UID SEARCH are different commands from FETCH, STORE, and SEARCH. An EXPUNGE response MAY be sent during a UID command. The update from the EXPUNGE response MUST be recorded by the client. Example: S: * 44 EXPUNGE 7.4.2. FETCH Response Contents: message data The FETCH response returns data about a message to the client. The data are pairs of data item names and their values in parentheses. This response occurs as the result of a FETCH or STORE command, as well as by unilateral server decision (e.g., flag updates). The current data items are: BODY A form of BODYSTRUCTURE without extension data. Crispin Standards Track [Page 73] RFC 3501 IMAPv4 March 2003 BODY[
]<> A string expressing the body contents of the specified section. The string SHOULD be interpreted by the client according to the content transfer encoding, body type, and subtype. If the origin octet is specified, this string is a substring of the entire body contents, starting at that origin octet. This means that BODY[]<0> MAY be truncated, but BODY[] is NEVER truncated. Note: The origin octet facility MUST NOT be used by a server in a FETCH response unless the client specifically requested it by means of a FETCH of a BODY[
]<> data item. 8-bit textual data is permitted if a [CHARSET] identifier is part of the body parameter parenthesized list for this section. Note that headers (part specifiers HEADER or MIME, or the header portion of a MESSAGE/RFC822 part), MUST be 7-bit; 8-bit characters are not permitted in headers. Note also that the [RFC-2822] delimiting blank line between the header and the body is not affected by header line subsetting; the blank line is always included as part of header data, except in the case of a message which has no body and no blank line. Non-textual data such as binary data MUST be transfer encoded into a textual form, such as BASE64, prior to being sent to the client. To derive the original binary data, the client MUST decode the transfer encoded string. BODYSTRUCTURE A parenthesized list that describes the [MIME-IMB] body structure of a message. This is computed by the server by parsing the [MIME-IMB] header fields, defaulting various fields as necessary. For example, a simple text message of 48 lines and 2279 octets can have a body structure of: ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 2279 48) Multiple parts are indicated by parenthesis nesting. Instead of a body type as the first element of the parenthesized list, there is a sequence of one or more nested body structures. The second element of the parenthesized list is the multipart subtype (mixed, digest, parallel, alternative, etc.). Crispin Standards Track [Page 74] RFC 3501 IMAPv4 March 2003 For example, a two part message consisting of a text and a BASE64-encoded text attachment can have a body structure of: (("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 1152 23)("TEXT" "PLAIN" ("CHARSET" "US-ASCII" "NAME" "cc.diff") "<960723163407.20117h@cac.washington.edu>" "Compiler diff" "BASE64" 4554 73) "MIXED") Extension data follows the multipart subtype. Extension data is never returned with the BODY fetch, but can be returned with a BODYSTRUCTURE fetch. Extension data, if present, MUST be in the defined order. The extension data of a multipart body part are in the following order: body parameter parenthesized list A parenthesized list of attribute/value pairs [e.g., ("foo" "bar" "baz" "rag") where "bar" is the value of "foo", and "rag" is the value of "baz"] as defined in [MIME-IMB]. body disposition A parenthesized list, consisting of a disposition type string, followed by a parenthesized list of disposition attribute/value pairs as defined in [DISPOSITION]. body language A string or parenthesized list giving the body language value as defined in [LANGUAGE-TAGS]. body location A string list giving the body content URI as defined in [LOCATION]. Any following extension data are not yet defined in this version of the protocol. Such extension data can consist of zero or more NILs, strings, numbers, or potentially nested parenthesized lists of such data. Client implementations that do a BODYSTRUCTURE fetch MUST be prepared to accept such extension data. Server implementations MUST NOT send such extension data until it has been defined by a revision of this protocol. The basic fields of a non-multipart body part are in the following order: body type A string giving the content media type name as defined in [MIME-IMB]. Crispin Standards Track [Page 75] RFC 3501 IMAPv4 March 2003 body subtype A string giving the content subtype name as defined in [MIME-IMB]. body parameter parenthesized list A parenthesized list of attribute/value pairs [e.g., ("foo" "bar" "baz" "rag") where "bar" is the value of "foo" and "rag" is the value of "baz"] as defined in [MIME-IMB]. body id A string giving the content id as defined in [MIME-IMB]. body description A string giving the content description as defined in [MIME-IMB]. body encoding A string giving the content transfer encoding as defined in [MIME-IMB]. body size A number giving the size of the body in octets. Note that this size is the size in its transfer encoding and not the resulting size after any decoding. A body type of type MESSAGE and subtype RFC822 contains, immediately after the basic fields, the envelope structure, body structure, and size in text lines of the encapsulated message. A body type of type TEXT contains, immediately after the basic fields, the size of the body in text lines. Note that this size is the size in its content transfer encoding and not the resulting size after any decoding. Extension data follows the basic fields and the type-specific fields listed above. Extension data is never returned with the BODY fetch, but can be returned with a BODYSTRUCTURE fetch. Extension data, if present, MUST be in the defined order. The extension data of a non-multipart body part are in the following order: body MD5 A string giving the body MD5 value as defined in [MD5]. Crispin Standards Track [Page 76] RFC 3501 IMAPv4 March 2003 body disposition A parenthesized list with the same content and function as the body disposition for a multipart body part. body language A string or parenthesized list giving the body language value as defined in [LANGUAGE-TAGS]. body location A string list giving the body content URI as defined in [LOCATION]. Any following extension data are not yet defined in this version of the protocol, and would be as described above under multipart extension data. ENVELOPE A parenthesized list that describes the envelope structure of a message. This is computed by the server by parsing the [RFC-2822] header into the component parts, defaulting various fields as necessary. The fields of the envelope structure are in the following order: date, subject, from, sender, reply-to, to, cc, bcc, in-reply-to, and message-id. The date, subject, in-reply-to, and message-id fields are strings. The from, sender, reply-to, to, cc, and bcc fields are parenthesized lists of address structures. An address structure is a parenthesized list that describes an electronic mail address. The fields of an address structure are in the following order: personal name, [SMTP] at-domain-list (source route), mailbox name, and host name. [RFC-2822] group syntax is indicated by a special form of address structure in which the host name field is NIL. If the mailbox name field is also NIL, this is an end of group marker (semi-colon in RFC 822 syntax). If the mailbox name field is non-NIL, this is a start of group marker, and the mailbox name field holds the group name phrase. If the Date, Subject, In-Reply-To, and Message-ID header lines are absent in the [RFC-2822] header, the corresponding member of the envelope is NIL; if these header lines are present but empty the corresponding member of the envelope is the empty string. Crispin Standards Track [Page 77] RFC 3501 IMAPv4 March 2003 Note: some servers may return a NIL envelope member in the "present but empty" case. Clients SHOULD treat NIL and empty string as identical. Note: [RFC-2822] requires that all messages have a valid Date header. Therefore, the date member in the envelope can not be NIL or the empty string. Note: [RFC-2822] requires that the In-Reply-To and Message-ID headers, if present, have non-empty content. Therefore, the in-reply-to and message-id members in the envelope can not be the empty string. If the From, To, cc, and bcc header lines are absent in the [RFC-2822] header, or are present but empty, the corresponding member of the envelope is NIL. If the Sender or Reply-To lines are absent in the [RFC-2822] header, or are present but empty, the server sets the corresponding member of the envelope to be the same value as the from member (the client is not expected to know to do this). Note: [RFC-2822] requires that all messages have a valid From header. Therefore, the from, sender, and reply-to members in the envelope can not be NIL. FLAGS A parenthesized list of flags that are set for this message. INTERNALDATE A string representing the internal date of the message. RFC822 Equivalent to BODY[]. RFC822.HEADER Equivalent to BODY[HEADER]. Note that this did not result in \Seen being set, because RFC822.HEADER response data occurs as a result of a FETCH of RFC822.HEADER. BODY[HEADER] response data occurs as a result of a FETCH of BODY[HEADER] (which sets \Seen) or BODY.PEEK[HEADER] (which does not set \Seen). RFC822.SIZE A number expressing the [RFC-2822] size of the message. Crispin Standards Track [Page 78] RFC 3501 IMAPv4 March 2003 RFC822.TEXT Equivalent to BODY[TEXT]. UID A number expressing the unique identifier of the message. Example: S: * 23 FETCH (FLAGS (\Seen) RFC822.SIZE 44827) 7.5. Server Responses - Command Continuation Request The command continuation request response is indicated by a "+" token instead of a tag. This form of response indicates that the server is ready to accept the continuation of a command from the client. The remainder of this response is a line of text. This response is used in the AUTHENTICATE command to transmit server data to the client, and request additional client data. This response is also used if an argument to any command is a literal. The client is not permitted to send the octets of the literal unless the server indicates that it is expected. This permits the server to process commands and reject errors on a line-by-line basis. The remainder of the command, including the CRLF that terminates a command, follows the octets of the literal. If there are any additional command arguments, the literal octets are followed by a space and those arguments. Example: C: A001 LOGIN {11} S: + Ready for additional command text C: FRED FOOBAR {7} S: + Ready for additional command text C: fat man S: A001 OK LOGIN completed C: A044 BLURDYBLOOP {102856} S: A044 BAD No such command as "BLURDYBLOOP" Crispin Standards Track [Page 79] RFC 3501 IMAPv4 March 2003 8. Sample IMAP4rev1 connection The following is a transcript of an IMAP4rev1 connection. A long line in this sample is broken for editorial clarity. S: * OK IMAP4rev1 Service Ready C: a001 login mrc secret S: a001 OK LOGIN completed C: a002 select inbox S: * 18 EXISTS S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * 2 RECENT S: * OK [UNSEEN 17] Message 17 is the first unseen message S: * OK [UIDVALIDITY 3857529045] UIDs valid S: a002 OK [READ-WRITE] SELECT completed C: a003 fetch 12 full S: * 12 FETCH (FLAGS (\Seen) INTERNALDATE "17-Jul-1996 02:44:25 -0700" RFC822.SIZE 4286 ENVELOPE ("Wed, 17 Jul 1996 02:23:25 -0700 (PDT)" "IMAP4rev1 WG mtg summary and minutes" (("Terry Gray" NIL "gray" "cac.washington.edu")) (("Terry Gray" NIL "gray" "cac.washington.edu")) (("Terry Gray" NIL "gray" "cac.washington.edu")) ((NIL NIL "imap" "cac.washington.edu")) ((NIL NIL "minutes" "CNRI.Reston.VA.US") ("John Klensin" NIL "KLENSIN" "MIT.EDU")) NIL NIL "") BODY ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 3028 92)) S: a003 OK FETCH completed C: a004 fetch 12 body[header] S: * 12 FETCH (BODY[HEADER] {342} S: Date: Wed, 17 Jul 1996 02:23:25 -0700 (PDT) S: From: Terry Gray S: Subject: IMAP4rev1 WG mtg summary and minutes S: To: imap@cac.washington.edu S: cc: minutes@CNRI.Reston.VA.US, John Klensin S: Message-Id: S: MIME-Version: 1.0 S: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII S: S: ) S: a004 OK FETCH completed C: a005 store 12 +flags \deleted S: * 12 FETCH (FLAGS (\Seen \Deleted)) S: a005 OK +FLAGS completed C: a006 logout S: * BYE IMAP4rev1 server terminating connection S: a006 OK LOGOUT completed Crispin Standards Track [Page 80] RFC 3501 IMAPv4 March 2003 9. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. In the case of alternative or optional rules in which a later rule overlaps an earlier rule, the rule which is listed earlier MUST take priority. For example, "\Seen" when parsed as a flag is the \Seen flag name and not a flag-extension, even though "\Seen" can be parsed as a flag-extension. Some, but not all, instances of this rule are noted below. Note: [ABNF] rules MUST be followed strictly; in particular: (1) Except as noted otherwise, all alphabetic characters are case-insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. (2) In all cases, SP refers to exactly one space. It is NOT permitted to substitute TAB, insert additional spaces, or otherwise treat SP as being equivalent to LWSP. (3) The ASCII NUL character, %x00, MUST NOT be used at any time. address = "(" addr-name SP addr-adl SP addr-mailbox SP addr-host ")" addr-adl = nstring ; Holds route from [RFC-2822] route-addr if ; non-NIL addr-host = nstring ; NIL indicates [RFC-2822] group syntax. ; Otherwise, holds [RFC-2822] domain name addr-mailbox = nstring ; NIL indicates end of [RFC-2822] group; if ; non-NIL and addr-host is NIL, holds ; [RFC-2822] group name. ; Otherwise, holds [RFC-2822] local-part ; after removing [RFC-2822] quoting Crispin Standards Track [Page 81] RFC 3501 IMAPv4 March 2003 addr-name = nstring ; If non-NIL, holds phrase from [RFC-2822] ; mailbox after removing [RFC-2822] quoting append = "APPEND" SP mailbox [SP flag-list] [SP date-time] SP literal astring = 1*ASTRING-CHAR / string ASTRING-CHAR = ATOM-CHAR / resp-specials atom = 1*ATOM-CHAR ATOM-CHAR = atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / quoted-specials / resp-specials authenticate = "AUTHENTICATE" SP auth-type *(CRLF base64) auth-type = atom ; Defined by [SASL] base64 = *(4base64-char) [base64-terminal] base64-char = ALPHA / DIGIT / "+" / "/" ; Case-sensitive base64-terminal = (2base64-char "==") / (3base64-char "=") body = "(" (body-type-1part / body-type-mpart) ")" body-extension = nstring / number / "(" body-extension *(SP body-extension) ")" ; Future expansion. Client implementations ; MUST accept body-extension fields. Server ; implementations MUST NOT generate ; body-extension fields except as defined by ; future standard or standards-track ; revisions of this specification. body-ext-1part = body-fld-md5 [SP body-fld-dsp [SP body-fld-lang [SP body-fld-loc *(SP body-extension)]]] ; MUST NOT be returned on non-extensible ; "BODY" fetch Crispin Standards Track [Page 82] RFC 3501 IMAPv4 March 2003 body-ext-mpart = body-fld-param [SP body-fld-dsp [SP body-fld-lang [SP body-fld-loc *(SP body-extension)]]] ; MUST NOT be returned on non-extensible ; "BODY" fetch body-fields = body-fld-param SP body-fld-id SP body-fld-desc SP body-fld-enc SP body-fld-octets body-fld-desc = nstring body-fld-dsp = "(" string SP body-fld-param ")" / nil body-fld-enc = (DQUOTE ("7BIT" / "8BIT" / "BINARY" / "BASE64"/ "QUOTED-PRINTABLE") DQUOTE) / string body-fld-id = nstring body-fld-lang = nstring / "(" string *(SP string) ")" body-fld-loc = nstring body-fld-lines = number body-fld-md5 = nstring body-fld-octets = number body-fld-param = "(" string SP string *(SP string SP string) ")" / nil body-type-1part = (body-type-basic / body-type-msg / body-type-text) [SP body-ext-1part] body-type-basic = media-basic SP body-fields ; MESSAGE subtype MUST NOT be "RFC822" body-type-mpart = 1*body SP media-subtype [SP body-ext-mpart] body-type-msg = media-message SP body-fields SP envelope SP body SP body-fld-lines body-type-text = media-text SP body-fields SP body-fld-lines capability = ("AUTH=" auth-type) / atom ; New capabilities MUST begin with "X" or be ; registered with IANA as standard or ; standards-track Crispin Standards Track [Page 83] RFC 3501 IMAPv4 March 2003 capability-data = "CAPABILITY" *(SP capability) SP "IMAP4rev1" *(SP capability) ; Servers MUST implement the STARTTLS, AUTH=PLAIN, ; and LOGINDISABLED capabilities ; Servers which offer RFC 1730 compatibility MUST ; list "IMAP4" as the first capability. CHAR8 = %x01-ff ; any OCTET except NUL, %x00 command = tag SP (command-any / command-auth / command-nonauth / command-select) CRLF ; Modal based on state command-any = "CAPABILITY" / "LOGOUT" / "NOOP" / x-command ; Valid in all states command-auth = append / create / delete / examine / list / lsub / rename / select / status / subscribe / unsubscribe ; Valid only in Authenticated or Selected state command-nonauth = login / authenticate / "STARTTLS" ; Valid only when in Not Authenticated state command-select = "CHECK" / "CLOSE" / "EXPUNGE" / copy / fetch / store / uid / search ; Valid only when in Selected state continue-req = "+" SP (resp-text / base64) CRLF copy = "COPY" SP sequence-set SP mailbox create = "CREATE" SP mailbox ; Use of INBOX gives a NO error date = date-text / DQUOTE date-text DQUOTE date-day = 1*2DIGIT ; Day of month date-day-fixed = (SP DIGIT) / 2DIGIT ; Fixed-format version of date-day date-month = "Jan" / "Feb" / "Mar" / "Apr" / "May" / "Jun" / "Jul" / "Aug" / "Sep" / "Oct" / "Nov" / "Dec" date-text = date-day "-" date-month "-" date-year Crispin Standards Track [Page 84] RFC 3501 IMAPv4 March 2003 date-year = 4DIGIT date-time = DQUOTE date-day-fixed "-" date-month "-" date-year SP time SP zone DQUOTE delete = "DELETE" SP mailbox ; Use of INBOX gives a NO error digit-nz = %x31-39 ; 1-9 envelope = "(" env-date SP env-subject SP env-from SP env-sender SP env-reply-to SP env-to SP env-cc SP env-bcc SP env-in-reply-to SP env-message-id ")" env-bcc = "(" 1*address ")" / nil env-cc = "(" 1*address ")" / nil env-date = nstring env-from = "(" 1*address ")" / nil env-in-reply-to = nstring env-message-id = nstring env-reply-to = "(" 1*address ")" / nil env-sender = "(" 1*address ")" / nil env-subject = nstring env-to = "(" 1*address ")" / nil examine = "EXAMINE" SP mailbox fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / "FAST" / fetch-att / "(" fetch-att *(SP fetch-att) ")") fetch-att = "ENVELOPE" / "FLAGS" / "INTERNALDATE" / "RFC822" [".HEADER" / ".SIZE" / ".TEXT"] / "BODY" ["STRUCTURE"] / "UID" / "BODY" section ["<" number "." nz-number ">"] / "BODY.PEEK" section ["<" number "." nz-number ">"] Crispin Standards Track [Page 85] RFC 3501 IMAPv4 March 2003 flag = "\Answered" / "\Flagged" / "\Deleted" / "\Seen" / "\Draft" / flag-keyword / flag-extension ; Does not include "\Recent" flag-extension = "\" atom ; Future expansion. Client implementations ; MUST accept flag-extension flags. Server ; implementations MUST NOT generate ; flag-extension flags except as defined by ; future standard or standards-track ; revisions of this specification. flag-fetch = flag / "\Recent" flag-keyword = atom flag-list = "(" [flag *(SP flag)] ")" flag-perm = flag / "\*" greeting = "*" SP (resp-cond-auth / resp-cond-bye) CRLF header-fld-name = astring header-list = "(" header-fld-name *(SP header-fld-name) ")" list = "LIST" SP mailbox SP list-mailbox list-mailbox = 1*list-char / string list-char = ATOM-CHAR / list-wildcards / resp-specials list-wildcards = "%" / "*" literal = "{" number "}" CRLF *CHAR8 ; Number represents the number of CHAR8s login = "LOGIN" SP userid SP password lsub = "LSUB" SP mailbox SP list-mailbox Crispin Standards Track [Page 86] RFC 3501 IMAPv4 March 2003 mailbox = "INBOX" / astring ; INBOX is case-insensitive. All case variants of ; INBOX (e.g., "iNbOx") MUST be interpreted as INBOX ; not as an astring. An astring which consists of ; the case-insensitive sequence "I" "N" "B" "O" "X" ; is considered to be INBOX and not an astring. ; Refer to section 5.1 for further ; semantic details of mailbox names. mailbox-data = "FLAGS" SP flag-list / "LIST" SP mailbox-list / "LSUB" SP mailbox-list / "SEARCH" *(SP nz-number) / "STATUS" SP mailbox SP "(" [status-att-list] ")" / number SP "EXISTS" / number SP "RECENT" mailbox-list = "(" [mbx-list-flags] ")" SP (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox mbx-list-flags = *(mbx-list-oflag SP) mbx-list-sflag *(SP mbx-list-oflag) / mbx-list-oflag *(SP mbx-list-oflag) mbx-list-oflag = "\Noinferiors" / flag-extension ; Other flags; multiple possible per LIST response mbx-list-sflag = "\Noselect" / "\Marked" / "\Unmarked" ; Selectability flags; only one per LIST response media-basic = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" / "MESSAGE" / "VIDEO") DQUOTE) / string) SP media-subtype ; Defined in [MIME-IMT] media-message = DQUOTE "MESSAGE" DQUOTE SP DQUOTE "RFC822" DQUOTE ; Defined in [MIME-IMT] media-subtype = string ; Defined in [MIME-IMT] media-text = DQUOTE "TEXT" DQUOTE SP media-subtype ; Defined in [MIME-IMT] message-data = nz-number SP ("EXPUNGE" / ("FETCH" SP msg-att)) msg-att = "(" (msg-att-dynamic / msg-att-static) *(SP (msg-att-dynamic / msg-att-static)) ")" msg-att-dynamic = "FLAGS" SP "(" [flag-fetch *(SP flag-fetch)] ")" ; MAY change for a message Crispin Standards Track [Page 87] RFC 3501 IMAPv4 March 2003 msg-att-static = "ENVELOPE" SP envelope / "INTERNALDATE" SP date-time / "RFC822" [".HEADER" / ".TEXT"] SP nstring / "RFC822.SIZE" SP number / "BODY" ["STRUCTURE"] SP body / "BODY" section ["<" number ">"] SP nstring / "UID" SP uniqueid ; MUST NOT change for a message nil = "NIL" nstring = string / nil number = 1*DIGIT ; Unsigned 32-bit integer ; (0 <= n < 4,294,967,296) nz-number = digit-nz *DIGIT ; Non-zero unsigned 32-bit integer ; (0 < n < 4,294,967,296) password = astring quoted = DQUOTE *QUOTED-CHAR DQUOTE QUOTED-CHAR = / "\" quoted-specials quoted-specials = DQUOTE / "\" rename = "RENAME" SP mailbox SP mailbox ; Use of INBOX as a destination gives a NO error response = *(continue-req / response-data) response-done response-data = "*" SP (resp-cond-state / resp-cond-bye / mailbox-data / message-data / capability-data) CRLF response-done = response-tagged / response-fatal response-fatal = "*" SP resp-cond-bye CRLF ; Server closes connection immediately response-tagged = tag SP resp-cond-state CRLF resp-cond-auth = ("OK" / "PREAUTH") SP resp-text ; Authentication condition Crispin Standards Track [Page 88] RFC 3501 IMAPv4 March 2003 resp-cond-bye = "BYE" SP resp-text resp-cond-state = ("OK" / "NO" / "BAD") SP resp-text ; Status condition resp-specials = "]" resp-text = ["[" resp-text-code "]" SP] text resp-text-code = "ALERT" / "BADCHARSET" [SP "(" astring *(SP astring) ")" ] / capability-data / "PARSE" / "PERMANENTFLAGS" SP "(" [flag-perm *(SP flag-perm)] ")" / "READ-ONLY" / "READ-WRITE" / "TRYCREATE" / "UIDNEXT" SP nz-number / "UIDVALIDITY" SP nz-number / "UNSEEN" SP nz-number / atom [SP 1*] search = "SEARCH" [SP "CHARSET" SP astring] 1*(SP search-key) ; CHARSET argument to MUST be registered with IANA search-key = "ALL" / "ANSWERED" / "BCC" SP astring / "BEFORE" SP date / "BODY" SP astring / "CC" SP astring / "DELETED" / "FLAGGED" / "FROM" SP astring / "KEYWORD" SP flag-keyword / "NEW" / "OLD" / "ON" SP date / "RECENT" / "SEEN" / "SINCE" SP date / "SUBJECT" SP astring / "TEXT" SP astring / "TO" SP astring / "UNANSWERED" / "UNDELETED" / "UNFLAGGED" / "UNKEYWORD" SP flag-keyword / "UNSEEN" / ; Above this line were in [IMAP2] "DRAFT" / "HEADER" SP header-fld-name SP astring / "LARGER" SP number / "NOT" SP search-key / "OR" SP search-key SP search-key / "SENTBEFORE" SP date / "SENTON" SP date / "SENTSINCE" SP date / "SMALLER" SP number / "UID" SP sequence-set / "UNDRAFT" / sequence-set / "(" search-key *(SP search-key) ")" section = "[" [section-spec] "]" section-msgtext = "HEADER" / "HEADER.FIELDS" [".NOT"] SP header-list / "TEXT" ; top-level or MESSAGE/RFC822 part section-part = nz-number *("." nz-number) ; body part nesting Crispin Standards Track [Page 89] RFC 3501 IMAPv4 March 2003 section-spec = section-msgtext / (section-part ["." section-text]) section-text = section-msgtext / "MIME" ; text other than actual body part (headers, etc.) select = "SELECT" SP mailbox seq-number = nz-number / "*" ; message sequence number (COPY, FETCH, STORE ; commands) or unique identifier (UID COPY, ; UID FETCH, UID STORE commands). ; * represents the largest number in use. In ; the case of message sequence numbers, it is ; the number of messages in a non-empty mailbox. ; In the case of unique identifiers, it is the ; unique identifier of the last message in the ; mailbox or, if the mailbox is empty, the ; mailbox's current UIDNEXT value. ; The server should respond with a tagged BAD ; response to a command that uses a message ; sequence number greater than the number of ; messages in the selected mailbox. This ; includes "*" if the selected mailbox is empty. seq-range = seq-number ":" seq-number ; two seq-number values and all values between ; these two regardless of order. ; Example: 2:4 and 4:2 are equivalent and indicate ; values 2, 3, and 4. ; Example: a unique identifier sequence range of ; 3291:* includes the UID of the last message in ; the mailbox, even if that value is less than 3291. sequence-set = (seq-number / seq-range) *("," sequence-set) ; set of seq-number values, regardless of order. ; Servers MAY coalesce overlaps and/or execute the ; sequence in any order. ; Example: a message sequence number set of ; 2,4:7,9,12:* for a mailbox with 15 messages is ; equivalent to 2,4,5,6,7,9,12,13,14,15 ; Example: a message sequence number set of *:4,5:7 ; for a mailbox with 10 messages is equivalent to ; 10,9,8,7,6,5,4,5,6,7 and MAY be reordered and ; overlap coalesced to be 4,5,6,7,8,9,10. status = "STATUS" SP mailbox SP "(" status-att *(SP status-att) ")" Crispin Standards Track [Page 90] RFC 3501 IMAPv4 March 2003 status-att = "MESSAGES" / "RECENT" / "UIDNEXT" / "UIDVALIDITY" / "UNSEEN" status-att-list = status-att SP number *(SP status-att SP number) store = "STORE" SP sequence-set SP store-att-flags store-att-flags = (["+" / "-"] "FLAGS" [".SILENT"]) SP (flag-list / (flag *(SP flag))) string = quoted / literal subscribe = "SUBSCRIBE" SP mailbox tag = 1* text = 1*TEXT-CHAR TEXT-CHAR = time = 2DIGIT ":" 2DIGIT ":" 2DIGIT ; Hours minutes seconds uid = "UID" SP (copy / fetch / search / store) ; Unique identifiers used instead of message ; sequence numbers uniqueid = nz-number ; Strictly ascending unsubscribe = "UNSUBSCRIBE" SP mailbox userid = astring x-command = "X" atom zone = ("+" / "-") 4DIGIT ; Signed four-digit value of hhmm representing ; hours and minutes east of Greenwich (that is, ; the amount that the given time differs from ; Universal Time). Subtracting the timezone ; from the given time will give the UT form. ; The Universal Time zone is "+0000". Crispin Standards Track [Page 91] RFC 3501 IMAPv4 March 2003 10. Author's Note This document is a revision or rewrite of earlier documents, and supercedes the protocol specification in those documents: RFC 2060, RFC 1730, unpublished IMAP2bis.TXT document, RFC 1176, and RFC 1064. 11. Security Considerations IMAP4rev1 protocol transactions, including electronic mail data, are sent in the clear over the network unless protection from snooping is negotiated. This can be accomplished either by the use of STARTTLS, negotiated privacy protection in the AUTHENTICATE command, or some other protection mechanism. 11.1. STARTTLS Security Considerations The specification of the STARTTLS command and LOGINDISABLED capability in this document replaces that in [IMAP-TLS]. [IMAP-TLS] remains normative for the PLAIN [SASL] authenticator. IMAP client and server implementations MUST implement the TLS_RSA_WITH_RC4_128_MD5 [TLS] cipher suite, and SHOULD implement the TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA [TLS] cipher suite. This is important as it assures that any two compliant implementations can be configured to interoperate. All other cipher suites are OPTIONAL. Note that this is a change from section 2.1 of [IMAP-TLS]. During the [TLS] negotiation, the client MUST check its understanding of the server hostname against the server's identity as presented in the server Certificate message, in order to prevent man-in-the-middle attacks. If the match fails, the client SHOULD either ask for explicit user confirmation, or terminate the connection and indicate that the server's identity is suspect. Matching is performed according to these rules: The client MUST use the server hostname it used to open the connection as the value to compare against the server name as expressed in the server certificate. The client MUST NOT use any form of the server hostname derived from an insecure remote source (e.g., insecure DNS lookup). CNAME canonicalization is not done. If a subjectAltName extension of type dNSName is present in the certificate, it SHOULD be used as the source of the server's identity. Matching is case-insensitive. Crispin Standards Track [Page 92] RFC 3501 IMAPv4 March 2003 A "*" wildcard character MAY be used as the left-most name component in the certificate. For example, *.example.com would match a.example.com, foo.example.com, etc. but would not match example.com. If the certificate contains multiple names (e.g., more than one dNSName field), then a match with any one of the fields is considered acceptable. Both the client and server MUST check the result of the STARTTLS command and subsequent [TLS] negotiation to see whether acceptable authentication or privacy was achieved. 11.2. Other Security Considerations A server error message for an AUTHENTICATE command which fails due to invalid credentials SHOULD NOT detail why the credentials are invalid. Use of the LOGIN command sends passwords in the clear. This can be avoided by using the AUTHENTICATE command with a [SASL] mechanism that does not use plaintext passwords, by first negotiating encryption via STARTTLS or some other protection mechanism. A server implementation MUST implement a configuration that, at the time of authentication, requires: (1) The STARTTLS command has been negotiated. OR (2) Some other mechanism that protects the session from password snooping has been provided. OR (3) The following measures are in place: (a) The LOGINDISABLED capability is advertised, and [SASL] mechanisms (such as PLAIN) using plaintext passwords are NOT advertised in the CAPABILITY list. AND (b) The LOGIN command returns an error even if the password is correct. AND (c) The AUTHENTICATE command returns an error with all [SASL] mechanisms that use plaintext passwords, even if the password is correct. A server error message for a failing LOGIN command SHOULD NOT specify that the user name, as opposed to the password, is invalid. A server SHOULD have mechanisms in place to limit or delay failed AUTHENTICATE/LOGIN attempts. Crispin Standards Track [Page 93] RFC 3501 IMAPv4 March 2003 Additional security considerations are discussed in the section discussing the AUTHENTICATE and LOGIN commands. 12. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/imap4-capabilities As this specification revises the STARTTLS and LOGINDISABLED extensions previously defined in [IMAP-TLS], the registry will be updated accordingly. Crispin Standards Track [Page 94] RFC 3501 IMAPv4 March 2003 Appendices A. Normative References The following documents contain definitions or specifications that are necessary to understand this document properly: [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [ANONYMOUS] Newman, C., "Anonymous SASL Mechanism", RFC 2245, November 1997. [CHARSET] Freed, N. and J. Postel, "IANA Character Set Registration Procedures", RFC 2978, October 2000. [DIGEST-MD5] Leach, P. and C. Newman, "Using Digest Authentication as a SASL Mechanism", RFC 2831, May 2000. [DISPOSITION] Troost, R., Dorner, S. and K. Moore, "Communicating Presentation Information in Internet Messages: The Content-Disposition Header", RFC 2183, August 1997. [IMAP-TLS] Newman, C., "Using TLS with IMAP, POP3 and ACAP", RFC 2595, June 1999. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [LANGUAGE-TAGS] Alvestrand, H., "Tags for the Identification of Languages", BCP 47, RFC 3066, January 2001. [LOCATION] Palme, J., Hopmann, A. and N. Shelness, "MIME Encapsulation of Aggregate Documents, such as HTML (MHTML)", RFC 2557, March 1999. [MD5] Myers, J. and M. Rose, "The Content-MD5 Header Field", RFC 1864, October 1995. Crispin Standards Track [Page 95] RFC 3501 IMAPv4 March 2003 [MIME-HDRS] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. [MIME-IMB] Freed, N. and N. Borenstein, "MIME (Multipurpose Internet Mail Extensions) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [MIME-IMT] Freed, N. and N. Borenstein, "MIME (Multipurpose Internet Mail Extensions) Part Two: Media Types", RFC 2046, November 1996. [RFC-2822] Resnick, P., "Internet Message Format", RFC 2822, April 2001. [SASL] Myers, J., "Simple Authentication and Security Layer (SASL)", RFC 2222, October 1997. [TLS] Dierks, T. and C. Allen, "The TLS Protocol Version 1.0", RFC 2246, January 1999. [UTF-7] Goldsmith, D. and M. Davis, "UTF-7: A Mail-Safe Transformation Format of Unicode", RFC 2152, May 1997. The following documents describe quality-of-implementation issues that should be carefully considered when implementing this protocol: [IMAP-IMPLEMENTATION] Leiba, B., "IMAP Implementation Recommendations", RFC 2683, September 1999. [IMAP-MULTIACCESS] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC 2180, July 1997. A.1 Informative References The following documents describe related protocols: [IMAP-DISC] Austein, R., "Synchronization Operations for Disconnected IMAP4 Clients", Work in Progress. [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail Models in IMAP4", RFC 1733, December 1994. Crispin Standards Track [Page 96] RFC 3501 IMAPv4 March 2003 [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [SMTP] Klensin, J., "Simple Mail Transfer Protocol", STD 10, RFC 2821, April 2001. The following documents are historical or describe historical aspects of this protocol: [IMAP-COMPAT] Crispin, M., "IMAP4 Compatibility with IMAP2bis", RFC 2061, December 1996. [IMAP-HISTORICAL] Crispin, M., "IMAP4 Compatibility with IMAP2 and IMAP2bis", RFC 1732, December 1994. [IMAP-OBSOLETE] Crispin, M., "Internet Message Access Protocol - Obsolete Syntax", RFC 2062, December 1996. [IMAP2] Crispin, M., "Interactive Mail Access Protocol - Version 2", RFC 1176, August 1990. [RFC-822] Crocker, D., "Standard for the Format of ARPA Internet Text Messages", STD 11, RFC 822, August 1982. [RFC-821] Postel, J., "Simple Mail Transfer Protocol", STD 10, RFC 821, August 1982. B. Changes from RFC 2060 1) Clarify description of unique identifiers and their semantics. 2) Fix the SELECT description to clarify that UIDVALIDITY is required in the SELECT and EXAMINE responses. 3) Added an example of a failing search. 4) Correct store-att-flags: "#flag" should be "1#flag". 5) Made search and section rules clearer. 6) Correct the STORE example. 7) Correct "BASE645" misspelling. 8) Remove extraneous close parenthesis in example of two-part message with text and BASE64 attachment. Crispin Standards Track [Page 97] RFC 3501 IMAPv4 March 2003 9) Remove obsolete "MAILBOX" response from mailbox-data. 10) A spurious "<" in the rule for mailbox-data was removed. 11) Add CRLF to continue-req. 12) Specifically exclude "]" from the atom in resp-text-code. 13) Clarify that clients and servers should adhere strictly to the protocol syntax. 14) Emphasize in 5.2 that EXISTS can not be used to shrink a mailbox. 15) Add NEWNAME to resp-text-code. 16) Clarify that the empty string, not NIL, is used as arguments to LIST. 17) Clarify that NIL can be returned as a hierarchy delimiter for the empty string mailbox name argument if the mailbox namespace is flat. 18) Clarify that addr-mailbox and addr-name have RFC-2822 quoting removed. 19) Update UTF-7 reference. 20) Fix example in 6.3.11. 21) Clarify that non-existent UIDs are ignored. 22) Update DISPOSITION reference. 23) Expand state diagram. 24) Clarify that partial fetch responses are only returned in response to a partial fetch command. 25) Add UIDNEXT response code. Correct UIDVALIDITY definition reference. 26) Further clarification of "can" vs. "MAY". 27) Reference RFC-2119. 28) Clarify that superfluous shifts are not permitted in modified UTF-7. 29) Clarify that there are no implicit shifts in modified UTF-7. Crispin Standards Track [Page 98] RFC 3501 IMAPv4 March 2003 30) Clarify that "INBOX" in a mailbox name is always INBOX, even if it is given as a string. 31) Add missing open parenthesis in media-basic grammar rule. 32) Correct attribute syntax in mailbox-data. 33) Add UIDNEXT to EXAMINE responses. 34) Clarify UNSEEN, PERMANENTFLAGS, UIDVALIDITY, and UIDNEXT responses in SELECT and EXAMINE. They are required now, but weren't in older versions. 35) Update references with RFC numbers. 36) Flush text-mime2. 37) Clarify that modified UTF-7 names must be case-sensitive and that violating the convention should be avoided. 38) Correct UID FETCH example. 39) Clarify UID FETCH, UID STORE, and UID SEARCH vs. untagged EXPUNGE responses. 40) Clarify the use of the word "convention". 41) Clarify that a command is not "in progress" until it has been fully received (specifically, that a command is not "in progress" during command continuation negotiation). 42) Clarify envelope defaulting. 43) Clarify that SP means one and only one space character. 44) Forbid silly states in LIST response. 45) Clarify that the ENVELOPE, INTERNALDATE, RFC822*, BODY*, and UID for a message is static. 46) Add BADCHARSET response code. 47) Update formal syntax to [ABNF] conventions. 48) Clarify trailing hierarchy delimiter in CREATE semantics. 49) Clarify that the "blank line" is the [RFC-2822] delimiting blank line. Crispin Standards Track [Page 99] RFC 3501 IMAPv4 March 2003 50) Clarify that RENAME should also create hierarchy as needed for the command to complete. 51) Fix body-ext-mpart to not require language if disposition present. 52) Clarify the RFC822.HEADER response. 53) Correct missing space after charset astring in search. 54) Correct missing quote for BADCHARSET in resp-text-code. 55) Clarify that ALL, FAST, and FULL preclude any other data items appearing. 56) Clarify semantics of reference argument in LIST. 57) Clarify that a null string for SEARCH HEADER X-FOO means any message with a header line with a field-name of X-FOO regardless of the text of the header. 58) Specifically reserve 8-bit mailbox names for future use as UTF-8. 59) It is not an error for the client to store a flag that is not in the PERMANENTFLAGS list; however, the server will either ignore the change or make the change in the session only. 60) Correct/clarify the text regarding superfluous shifts. 61) Correct typographic errors in the "Changes" section. 62) Clarify that STATUS must not be used to check for new messages in the selected mailbox 63) Clarify LSUB behavior with "%" wildcard. 64) Change AUTHORIZATION to AUTHENTICATE in section 7.5. 65) Clarify description of multipart body type. 66) Clarify that STORE FLAGS does not affect \Recent. 67) Change "west" to "east" in description of timezone. 68) Clarify that commands which break command pipelining must wait for a completion result response. 69) Clarify that EXAMINE does not affect \Recent. Crispin Standards Track [Page 100] RFC 3501 IMAPv4 March 2003 70) Make description of MIME structure consistent. 71) Clarify that date searches disregard the time and timezone of the INTERNALDATE or Date: header. In other words, "ON 13-APR-2000" means messages with an INTERNALDATE text which starts with "13-APR-2000", even if timezone differential from the local timezone is sufficient to move that INTERNALDATE into the previous or next day. 72) Clarify that the header fetches don't add a blank line if one isn't in the [RFC-2822] message. 73) Clarify (in discussion of UIDs) that messages are immutable. 74) Add an example of CHARSET searching. 75) Clarify in SEARCH that keywords are a type of flag. 76) Clarify the mandatory nature of the SELECT data responses. 77) Add optional CAPABILITY response code in the initial OK or PREAUTH. 78) Add note that server can send an untagged CAPABILITY command as part of the responses to AUTHENTICATE and LOGIN. 79) Remove statement about it being unnecessary to issue a CAPABILITY command more than once in a connection. That statement is no longer true. 80) Clarify that untagged EXPUNGE decrements the number of messages in the mailbox. 81) Fix definition of "body" (concatenation has tighter binding than alternation). 82) Add a new "Special Notes to Implementors" section with reference to [IMAP-IMPLEMENTATION]. 83) Clarify that an untagged CAPABILITY response to an AUTHENTICATE command should only be done if a security layer was not negotiated. 84) Change the definition of atom to exclude "]". Update astring to include "]" for compatibility with the past. Remove resp-text-atom. 85) Remove NEWNAME. It can't work because mailbox names can be literals and can include "]". Functionality can be addressed via referrals. Crispin Standards Track [Page 101] RFC 3501 IMAPv4 March 2003 86) Move modified UTF-7 rationale in order to have more logical paragraph flow. 87) Clarify UID uniqueness guarantees with the use of MUST. 88) Note that clients should read response data until the connection is closed instead of immediately closing on a BYE. 89) Change RFC-822 references to RFC-2822. 90) Clarify that RFC-2822 should be followed instead of RFC-822. 91) Change recommendation of optional automatic capabilities in LOGIN and AUTHENTICATE to use the CAPABILITY response code in the tagged OK. This is more interoperable than an unsolicited untagged CAPABILITY response. 92) STARTTLS and AUTH=PLAIN are mandatory to implement; add recommendations for other [SASL] mechanisms. 93) Clarify that a "connection" (as opposed to "server" or "command") is in one of the four states. 94) Clarify that a failed or rejected command does not change state. 95) Split references between normative and informative. 96) Discuss authentication failure issues in security section. 97) Clarify that a data item is not necessarily of only one data type. 98) Clarify that sequence ranges are independent of order. 99) Change an example to clarify that superfluous shifts in Modified-UTF7 can not be fixed just by omitting the shift. The entire string must be recalculated. 100) Change Envelope Structure definition since [RFC-2822] uses "envelope" to refer to the [SMTP] envelope and not the envelope data that appears in the [RFC-2822] header. 101) Expand on RFC822.HEADER response data vs. BODY[HEADER]. 102) Clarify Logout state semantics, change ASCII art. 103) Security changes to comply with IESG requirements. Crispin Standards Track [Page 102] RFC 3501 IMAPv4 March 2003 104) Add definition for body URI. 105) Break sequence range definition into three rules, with rewritten descriptions for each. 106) Move STARTTLS and LOGINDISABLED here from [IMAP-TLS]. 107) Add IANA Considerations section. 108) Clarify valid client assumptions for new message UIDs vs. UIDNEXT. 109) Clarify that changes to permanentflags affect concurrent sessions as well as subsequent sessions. 110) Clarify that authenticated state can be entered by the CLOSE command. 111) Emphasize that SELECT and EXAMINE are the exceptions to the rule that a failing command does not change state. 112) Clarify that newly-appended messages have the Recent flag set. 113) Clarify that newly-copied messages SHOULD have the Recent flag set. 114) Clarify that UID commands always return the UID in FETCH responses. C. Key Word Index +FLAGS (store command data item) ............... 59 +FLAGS.SILENT (store command data item) ........ 59 -FLAGS (store command data item) ............... 59 -FLAGS.SILENT (store command data item) ........ 59 ALERT (response code) ...................................... 64 ALL (fetch item) ........................................... 55 ALL (search key) ........................................... 50 ANSWERED (search key) ...................................... 50 APPEND (command) ........................................... 45 AUTHENTICATE (command) ..................................... 27 BAD (response) ............................................. 66 BADCHARSET (response code) ................................. 64 BCC (search key) .................................. 51 BEFORE (search key) ................................. 51 BODY (fetch item) .......................................... 55 BODY (fetch result) ........................................ 73 BODY (search key) ................................. 51 Crispin Standards Track [Page 103] RFC 3501 IMAPv4 March 2003 BODY.PEEK[
]<> (fetch item) ............... 57 BODYSTRUCTURE (fetch item) ................................. 57 BODYSTRUCTURE (fetch result) ............................... 74 BODY[
]<> (fetch result) ............. 74 BODY[
]<> (fetch item) .................... 55 BYE (response) ............................................. 67 Body Structure (message attribute) ......................... 12 CAPABILITY (command) ....................................... 24 CAPABILITY (response code) ................................. 64 CAPABILITY (response) ...................................... 68 CC (search key) ................................... 51 CHECK (command) ............................................ 47 CLOSE (command) ............................................ 48 COPY (command) ............................................. 59 CREATE (command) ........................................... 34 DELETE (command) ........................................... 35 DELETED (search key) ....................................... 51 DRAFT (search key) ......................................... 51 ENVELOPE (fetch item) ...................................... 57 ENVELOPE (fetch result) .................................... 77 EXAMINE (command) .......................................... 33 EXISTS (response) .......................................... 71 EXPUNGE (command) .......................................... 48 EXPUNGE (response) ......................................... 72 Envelope Structure (message attribute) ..................... 12 FAST (fetch item) .......................................... 55 FETCH (command) ............................................ 54 FETCH (response) ........................................... 73 FLAGGED (search key) ....................................... 51 FLAGS (fetch item) ......................................... 57 FLAGS (fetch result) ....................................... 78 FLAGS (response) ........................................... 71 FLAGS (store command data item) ................ 59 FLAGS.SILENT (store command data item) ......... 59 FROM (search key) ................................. 51 FULL (fetch item) .......................................... 55 Flags (message attribute) .................................. 11 HEADER (part specifier) .................................... 55 HEADER (search key) .................. 51 HEADER.FIELDS (part specifier) ............... 55 HEADER.FIELDS.NOT (part specifier) ........... 55 INTERNALDATE (fetch item) .................................. 57 INTERNALDATE (fetch result) ................................ 78 Internal Date (message attribute) .......................... 12 KEYWORD (search key) ................................ 51 Keyword (type of flag) ..................................... 11 LARGER (search key) .................................... 51 LIST (command) ............................................. 40 Crispin Standards Track [Page 104] RFC 3501 IMAPv4 March 2003 LIST (response) ............................................ 69 LOGIN (command) ............................................ 30 LOGOUT (command) ........................................... 25 LSUB (command) ............................................. 43 LSUB (response) ............................................ 70 MAY (specification requirement term) ....................... 4 MESSAGES (status item) ..................................... 45 MIME (part specifier) ...................................... 56 MUST (specification requirement term) ...................... 4 MUST NOT (specification requirement term) .................. 4 Message Sequence Number (message attribute) ................ 10 NEW (search key) ........................................... 51 NO (response) .............................................. 66 NOOP (command) ............................................. 25 NOT (search key) .............................. 52 OK (response) .............................................. 65 OLD (search key) ........................................... 52 ON (search key) ..................................... 52 OPTIONAL (specification requirement term) .................. 4 OR (search key) ................ 52 PARSE (response code) ...................................... 64 PERMANENTFLAGS (response code) ............................. 64 PREAUTH (response) ......................................... 67 Permanent Flag (class of flag) ............................. 12 READ-ONLY (response code) .................................. 65 READ-WRITE (response code) ................................. 65 RECENT (response) .......................................... 72 RECENT (search key) ........................................ 52 RECENT (status item) ....................................... 45 RENAME (command) ........................................... 37 REQUIRED (specification requirement term) .................. 4 RFC822 (fetch item) ........................................ 57 RFC822 (fetch result) ...................................... 78 RFC822.HEADER (fetch item) ................................. 57 RFC822.HEADER (fetch result) ............................... 78 RFC822.SIZE (fetch item) ................................... 57 RFC822.SIZE (fetch result) ................................. 78 RFC822.TEXT (fetch item) ................................... 58 RFC822.TEXT (fetch result) ................................. 79 SEARCH (command) ........................................... 49 SEARCH (response) .......................................... 71 SEEN (search key) .......................................... 52 SELECT (command) ........................................... 31 SENTBEFORE (search key) ............................. 52 SENTON (search key) ................................. 52 SENTSINCE (search key) .............................. 52 SHOULD (specification requirement term) .................... 4 SHOULD NOT (specification requirement term) ................ 4 Crispin Standards Track [Page 105] RFC 3501 IMAPv4 March 2003 SINCE (search key) .................................. 52 SMALLER (search key) ................................... 52 STARTTLS (command) ......................................... 27 STATUS (command) ........................................... 44 STATUS (response) .......................................... 70 STORE (command) ............................................ 58 SUBJECT (search key) .............................. 53 SUBSCRIBE (command) ........................................ 38 Session Flag (class of flag) ............................... 12 System Flag (type of flag) ................................. 11 TEXT (part specifier) ...................................... 56 TEXT (search key) ................................. 53 TO (search key) ................................... 53 TRYCREATE (response code) .................................. 65 UID (command) .............................................. 60 UID (fetch item) ........................................... 58 UID (fetch result) ......................................... 79 UID (search key) ............................ 53 UIDNEXT (response code) .................................... 65 UIDNEXT (status item) ...................................... 45 UIDVALIDITY (response code) ................................ 65 UIDVALIDITY (status item) .................................. 45 UNANSWERED (search key) .................................... 53 UNDELETED (search key) ..................................... 53 UNDRAFT (search key) ....................................... 53 UNFLAGGED (search key) ..................................... 53 UNKEYWORD (search key) .............................. 53 UNSEEN (response code) ..................................... 65 UNSEEN (search key) ........................................ 53 UNSEEN (status item) ....................................... 45 UNSUBSCRIBE (command) ...................................... 39 Unique Identifier (UID) (message attribute) ................ 8 X (command) .......................................... 62 [RFC-2822] Size (message attribute) ........................ 12 \Answered (system flag) .................................... 11 \Deleted (system flag) ..................................... 11 \Draft (system flag) ....................................... 11 \Flagged (system flag) ..................................... 11 \Marked (mailbox name attribute) ........................... 69 \Noinferiors (mailbox name attribute) ...................... 69 \Noselect (mailbox name attribute) ......................... 69 \Recent (system flag) ...................................... 11 \Seen (system flag) ........................................ 11 \Unmarked (mailbox name attribute) ......................... 69 Crispin Standards Track [Page 106] RFC 3501 IMAPv4 March 2003 Author's Address Mark R. Crispin Networks and Distributed Computing University of Washington 4545 15th Avenue NE Seattle, WA 98105-4527 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin Standards Track [Page 107] RFC 3501 IMAPv4 March 2003 Full Copyright Statement Copyright (C) The Internet Society (2003). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. v This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Crispin Standards Track [Page 108] ================================================ FILE: docs/rfcs/rfc3502.MULTIAPPEND_extension.txt ================================================ Network Working Group M. Crispin Request for Comments: 3502 University of Washington Category: Standards Track March 2003 Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2003). All Rights Reserved. Abstract This document describes the multiappending extension to the Internet Message Access Protocol (IMAP) (RFC 3501). This extension provides substantial performance improvements for IMAP clients which upload multiple messages at a time to a mailbox on the server. A server which supports this extension indicates this with a capability name of "MULTIAPPEND". Terminology The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. Introduction The MULTIAPPEND extension permits uploading of multiple messages with a single command. When used in conjunction with the [LITERAL+] extension, the entire upload is accomplished in a single command/response round trip. A MULTIAPPEND APPEND operation is atomic; either all messages are successfully appended, or no messages are appended. In the base IMAP specification, each message must be appended in a separate command, and there is no mechanism to "unappend" messages if an error occurs while appending. Also, some mail stores may require Crispin Standards Track [Page 1] RFC 3502 IMAP MULTIAPPEND March 2003 an expensive "open/lock + sync/unlock/close" operation as part of appending; this can be quite expensive if it must be done on a per-message basis. If the server supports both LITERAL+ and pipelining but not MULTIAPPEND, it may be possible to get some of the performance advantages of MULTIAPPEND by doing a pipelined "batch" append. However, it will not work as well as MULTIAPPEND for the following reasons: 1) Multiple APPEND commands, even as part of a pipelined batch, are non-atomic by definition. There is no way to revert the mailbox to the state before the batch append in the event of an error. 2) It may not be feasible for the server to coalesce pipelined APPEND operations so as to avoid the "open/lock + sync/unlock/close" overhead described above. In any case, such coalescing would be timing dependent and thus potentially unreliable. In particular, with traditional UNIX mailbox files, it is assumed that a lock is held only for a single atomic operation, and many applications disregard any lock that is older than 5 minutes. 3) If an error occurs, depending upon the nature of the error, it is possible for additional messages to be appended after the error. For example, the user wants to append 5 messages, but a disk quota error occurs with the third message because of its size. However, the fourth and fifth messages have already been sent in the pipeline, so the mailbox ends up with the first, second, fourth, and fifth messages of the batch appended. 6.3.11. APPEND Command Arguments: mailbox name one or more messages to upload, specified as: OPTIONAL flag parenthesized list OPTIONAL date/time string message literal Data: no specific responses for this command Result: OK - append completed NO - append error: can't append to that mailbox, error in flags or date/time or message text, append cancelled BAD - command unknown or arguments invalid Crispin Standards Track [Page 2] RFC 3502 IMAP MULTIAPPEND March 2003 The APPEND command appends the literal arguments as new messages to the end of the specified destination mailbox. This argument SHOULD be in the format of an [RFC-2822] message. 8-bit characters are permitted in the message. A server implementation that is unable to preserve 8-bit data properly MUST be able to reversibly convert 8-bit APPEND data to 7-bit using a [MIME-IMB] content transfer encoding. Note: There MAY be exceptions, e.g., draft messages, in which required [RFC-2822] header lines are omitted in the message literal argument to APPEND. The full implications of doing so MUST be understood and carefully weighed. If a flag parenthesized list is specified, the flags SHOULD be set in the resulting message; otherwise, the flag list of the resulting message is set empty by default. If a date-time is specified, the internal date SHOULD be set in the resulting message; otherwise, the internal date of the resulting message is set to the current date and time by default. A zero-length message literal argument is an error, and MUST return a NO. This can be used to cancel the append. If the append is unsuccessful for any reason (including being cancelled), the mailbox MUST be restored to its state before the APPEND attempt; no partial appending is permitted. The server MAY return an error before processing all the message arguments. If the destination mailbox does not exist, a server MUST return an error, and MUST NOT automatically create the mailbox. Unless it is certain that the destination mailbox can not be created, the server MUST send the response code "[TRYCREATE]" as the prefix of the text of the tagged NO response. This gives a hint to the client that it can attempt a CREATE command and retry the APPEND if the CREATE is successful. If the mailbox is currently selected, the normal new message actions SHOULD occur. Specifically, the server SHOULD notify the client immediately via an untagged EXISTS response. If the server does not do so, the client MAY issue a NOOP command (or failing that, a CHECK command) after one or more APPEND commands. Crispin Standards Track [Page 3] RFC 3502 IMAP MULTIAPPEND March 2003 Example: C: A003 APPEND saved-messages (\Seen) {329} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.example.net C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: (\Seen) " 7-Feb-1994 22:43:04 -0800" {295} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) C: From: Joe Mooch C: Subject: Re: afternoon meeting C: To: foobar@blurdybloop.example.com C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: 3:30 is fine with me. C: S: A003 OK APPEND completed C: A004 APPEND bogusname (\Flagged) {1023} S: A004 NO [TRYCREATE] No such mailbox as bogusname C: A005 APPEND test (\Flagged) {99} S: + Ready for literal data C: Date: Mon, 7 Feb 2000 22:43:04 -0800 (PST) C: From: Fred Foobar C: Subject: hmm... C: {35403} S: A005 NO APPEND failed: Disk quota exceeded Note: The APPEND command is not used for message delivery, because it does not provide a mechanism to transfer [SMTP] envelope information. Modification to IMAP4rev1 Base Protocol Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. append = "APPEND" SP mailbox 1*append-message append-message = [SP flag-list] [SP date-time] SP literal Crispin Standards Track [Page 4] RFC 3502 IMAP MULTIAPPEND March 2003 MULTIAPPEND Interaction with UIDPLUS Extension Servers which support both MULTIAPPEND and [UIDPLUS] will have the "resp-code-apnd" rule modified as follows: resp-code-apnd = "APPENDUID" SP nz-number SP set That is, the APPENDUID response code returns as many UIDs as there were messages appended in the multiple append. The UIDs returned should be in the order the articles where appended. The message set may not contain extraneous UIDs or the symbol "*". Security Considerations The MULTIAPPEND extension does not raise any security considerations that are not present in the base [IMAP] protocol, and these issues are discussed in [IMAP]. Nevertheless, it is important to remember that IMAP4rev1 protocol transactions, including electronic mail data, are sent in the clear over the network unless protection from snooping is negotiated, either by the use of STARTTLS, privacy protection is negotiated in the AUTHENTICATE command, or some other protection mechanism is in effect. Normative References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [IMAP] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MIME-IMB] Freed, N. and N. Borenstein, "MIME (Multipurpose Internet Mail Extensions) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [RFC-2822] Resnick, P., "Internet Message Format", RFC 2822, April 2001. Crispin Standards Track [Page 5] RFC 3502 IMAP MULTIAPPEND March 2003 Informative References [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, January 1997. [UIDPLUS] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June 1988. [SMTP] Klensin, J., Editor, "Simple Mail Transfer Protocol", RFC 2821, April 2001. Author's Address Mark R. Crispin Networks and Distributed Computing University of Washington 4545 15th Avenue NE Seattle, WA 98105-4527 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin Standards Track [Page 6] RFC 3502 IMAP MULTIAPPEND March 2003 Full Copyright Statement Copyright (C) The Internet Society (2003). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Crispin Standards Track [Page 7] ================================================ FILE: docs/rfcs/rfc3503.Message_Disposition_Notification.txt ================================================ Network Working Group A. Melnikov Request for Comments: 3503 ACI Worldwide/MessagingDirect Category: Standards Track March 2003 Message Disposition Notification (MDN) profile for Internet Message Access Protocol (IMAP) Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2003). All Rights Reserved. Abstract The Message Disposition Notification (MDN) facility defined in RFC 2298 provides a means by which a message can request that message processing by the recipient be acknowledged as well as a format to be used for such acknowledgements. However, it doesn't describe how multiple Mail User Agents (MUAs) should handle the generation of MDNs in an Internet Message Access Protocol (IMAP4) environment. This document describes how to handle MDNs in such an environment and provides guidelines for implementers of IMAP4 that want to add MDN support to their products. Melnikov Standards Track [Page 1] RFC 3503 MDN profile for IMAP March 2003 Table of Contents 1. Conventions Used in this Document............................. 2 2. Introduction and Overview..................................... 2 3. Client behavior............................................... 3 3.1. Client behavior when receiving a message................. 5 3.2. Client behavior when copying a message................... 5 3.3. Client behavior when sending a message................... 5 3.4. Client behavior when saving a temporary message.......... 5 4. Server behavior............................................... 5 4.1. Server that supports arbitrary keywords.................. 5 4.2. Server that supports only $MDNSent keyword............... 5 4.3. Interaction with IMAP ACL extension...................... 6 5. Examples...................................................... 6 6. Security Considerations....................................... 7 7. Formal Syntax................................................. 7 8. Acknowledgments............................................... 7 9. Normative References.......................................... 8 10. Author's Address.............................................. 8 11. Full Copyright Statement...................................... 9 1. Conventions Used in this Document "C:" and "S:" in examples show lines sent by the client and server respectively. The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document when typed in uppercase are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 2. Introduction and Overview This memo defines an additional [IMAP4] mailbox keyword that allows multiple Mail User Agents (MUAs) to know if a requested receipt notification was sent. Message Disposition Notification [MDN] does not require any special support of IMAP in the case where a user has access to the mailstore from only one computer and is using a single MUA. In this case, the MUA behaves as described in [MDN], i.e., the MUA performs automatic processing and generates corresponding MDNs, it performs requested action and, with the user's permission, sends appropriate MDNs. The MUA will not send MDN twice because the MUA keeps track of sent notifications in a local configuration. However, that does not work when IMAP is used to access the same mailstore from different locations or is using different MUAs. Melnikov Standards Track [Page 2] RFC 3503 MDN profile for IMAP March 2003 This document defines a new special purpose mailbox keyword $MDNSent that must be used by MUAs. It does not define any new command or response for IMAP, but describes a technique that MUAs should use to achieve interoperability. When a client opens a mailbox for the first time, it verifies that the server is capable of storing the $MDNSent keyword by examining the PERMANENTFLAGS response code. In order to support MDN in IMAP, a server MUST support either the $MDNSent keyword, or arbitrary message keywords. 3. Client behavior The use of IMAP requires few additional steps in mail processing on the client side. The following timeline modifies the timeline found in Section 4 of [MDN]. -- User composes message. -- User tells MUA to send message. -- MUA passes message to MSA (original recipient information passed along). MUA [optionally] saves message to a folder for sent mail with $MDNSent flag set. -- MSA sends message to MTA. -- Final MTA receives message. -- Final MTA delivers message to MUA (possibly generating DSN). -- MUA logs into IMAP server, opens mailbox, verifies if mailbox can store $MDNSent keyword by examining PERMANENTFLAGS response. -- MUA performs automatic processing and generates corresponding MDNs ("dispatched", "processed", "deleted", "denied" or "failed" disposition type with "automatic-action" and "MDN-sent- automatically" disposition modes) for messages that do not have $MDNSent keyword, or \Draft flag set. (*) -- MUA sets the $MDNSent keyword for every message that required an automatic MDN to be sent, whether or not the MDN was sent. -- MUA displays a list of messages to user. -- User selects a message and requests that some action be performed on it. Melnikov Standards Track [Page 3] RFC 3503 MDN profile for IMAP March 2003 -- MUA performs requested action and, with user's permission, sends appropriate MDN ("displayed", "dispatched", "processed", "deleted", "denied" or "failed" disposition type with "manual- action" and "MDN-sent-manually" or "MDN-sent-automatically" disposition mode). If the generated MDN is saved to a mailbox with the APPEND command, the client MUST specify the $MDNSent keyword in the APPEND. -- MUA sets the $MDNSent keyword for all messages for which the user confirmed the dispatching of disposition (or was explicitly prohibited to do so). -- User possibly performs other actions on message, but no further MDNs are generated. (*) Note: MUA MUST NOT use \Recent flag as an indicator that it should send MDN, because according to [IMAP4], "If multiple connections have the same mailbox selected simultaneously, it is undefined which of these connections will see newly-arrived messages with \Recent set and which will see it without \Recent set". Thus, using \Recent as an indicator will cause unpredictable client behavior with different IMAP4 servers. However, the client MAY use \Seen flag as one of the indicators that MDN must not be sent. The client MUST NOT use any other standard flags, like \Draft or \Answered, to indicate that MDN was previously sent, because they have different well known meaning. In any case, in the presence of the $MDNSent keyword, the client MUST ignore all other flags or keywords for the purpose of generating an MDN and MUST NOT send the MDN. When the client opens a mailbox for the first time, it must verify that the server supports the $MDNSent keyword, or arbitrary message keywords by examining PERMANENTFLAGS response code. The client MUST NOT try to set the $MDNSent keyword if the server is incapable of storing it permanently. The client MUST be prepared to receive NO from the server as the result of STORE $MDNSent when the server advertises the support of storing arbitrary keywords, because the server may limit the number of message keywords it can store in a particular mailbox. A client SHOULD NOT send MDN if it fails to store the $MDNSent keyword. Once the $MDNSent keyword is set, it MUST NOT be unset by a client. The client MAY set the $MDNSent keyword when a user denies sending the notification. This prohibits all other MUAs from sending MDN for this message. Melnikov Standards Track [Page 4] RFC 3503 MDN profile for IMAP March 2003 3.1. Client behavior when receiving a message The client MUST NOT send MDN if a message has the $MDNSent keyword set. It also MUST NOT send MDN if a message has \Draft flag, because some clients use this flag to mark a message as incomplete. See the timeline in section 3 for details on client behavior when receiving a message. 3.2. Client behavior when copying a message The client SHOULD verify that $MDNSent is preserved on a COPY operation. Furthermore, when a message is copied between servers with the APPEND command, the client MUST set the $MDNSent keyword correctly. 3.3. Client behavior when sending a message When saving a sent message to any folder, the client MUST set the $MDNSent keyword to prevent another client from sending MDN for the message. 3.4. Client behavior when saving a temporary message When saving an unfinished message to any folder client MUST set $MDNSent keyword to prevent another client from sending MDN for the message. 4. Server behavior Server implementors that want to follow this specification must insure that their server complies with either section 4.1 or section 4.2. If the server also supports the IMAP [ACL] extension, it MUST also comply with the section 4.3. 4.1. Server that supports arbitrary keywords No changes are required from the server to make it compatible with the extension described in this document if it supports arbitrary keywords. 4.2. Server that supports only $MDNSent keyword Servers that support only the $MDNSent keyword MUST preserve it on the COPY operation. It is also expected that a server that supports SEARCH will also support the SEARCH KEYWORD $MDNSent. Melnikov Standards Track [Page 5] RFC 3503 MDN profile for IMAP March 2003 4.3. Interaction with IMAP ACL extension Any server that conforms to either 4.1 or 4.2 and also supports the IMAP [ACL] extension, SHOULD preserve the $MDNSent keyword on COPY even if the client does not have 'w' right. This will prevent the generation of a duplicated MDN for the same message. Note that the server MUST still check if the client has rights to perform the COPY operation on a message according to [ACL]. 5. Examples 1) MUA opens mailbox for the first time. a) The server supports storing of arbitrary keywords C: a100 select INBOX S: * FLAGS (\Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen \*)] S: * 5 EXISTS S: * 3 RECENT S: * OK [UIDVALIDITY 894294713] S: a100 OK [READ-WRITE] Completed b) The server supports storing of the $MDNSent keyword C: a100 select INBOX S: * FLAGS (\Flagged \Draft \Deleted \Seen $MDNSent) S: * OK [PERMANENTFLAGS (\Flagged \Draft \Deleted \Seen $MDNSent)] S: * 5 EXISTS S: * 3 RECENT S: * OK [UIDVALIDITY 894294713] S: a100 OK [READ-WRITE] Completed 2) The MUA successfully sets the $MDNSent keyword C: a200 STORE 4 +FLAGS ($MDNSent) S: * 4 FETCH (FLAGS (\Flagged \Seen $MDNSent)) S: * FLAGS ($MDNSent \Flagged \Deleted \Draft \Seen) S: * OK [PERMANENTFLAGS ($MDNSent \Flagged \Deleted \Draft \Seen \*)] S: a200 OK STORE completed 3) The server refuses to store the $MDNSent keyword C: a200 STORE 4 +FLAGS ($MDNSent) S: a200 NO STORE failed : no space left to store $MDNSent keyword Melnikov Standards Track [Page 6] RFC 3503 MDN profile for IMAP March 2003 4) All clients and servers MUST treat the $MDNSent keyword as case insensitive in all operations, as stated in [IMAP]. C: a300 FETCH 1:* FLAGS S: * 1 FETCH (FLAGS (\Seen)) S: * 2 FETCH (FLAGS (\Answered \Seen $MdnSENt)) S: * 3 FETCH (FLAGS ()) S: * 4 FETCH (FLAGS (\Flagged \Seen $MdnSENT)) S: * 5 FETCH (FLAGS ($MDNSent)) S: * 6 FETCH (FLAGS (\Recent)) S: a300 OK FETCH completed C: a400 SEARCH KEYWORDS $mdnsent S: * SEARCH 2 4 5 S: a400 OK SEARCH completed 6. Security Considerations There are no known security issues with this extension, not found in [MDN] and/or [IMAP4]. Section 4.3 changes ACL checking requirements on an IMAP server that implements IMAP [ACL] extension. 7. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) notation as specified in [RFC-822], as modified by [IMAP4]. Non-terminals referenced, but not defined below, are as defined by [IMAP4]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. flag_keyword ::= "$MDNSent" / other_keywords other_keywords ::= atom 8. Acknowledgments This document is the product of discussions that took place on the IMAP mailing list. Special gratitude to Cyrus Daboo and Randall Gellens for reviewing the document. Thank you to my father who as he has helped to make me what I am. I miss you terribly. Melnikov Standards Track [Page 7] RFC 3503 MDN profile for IMAP March 2003 9. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MDN] Fajman, R., "An Extensible Message Format for Message Disposition Notifications", RFC 2298, March 1998. [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [ACL] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. 10. Author's Address Alexey Melnikov ACI Worldwide/MessagingDirect 59 Clarendon Road Watford, Hertfordshire United Kingdom, WD17 1FQ Phone: +44 1923 81 2877 EMail: mel@messagingdirect.com Melnikov Standards Track [Page 8] RFC 3503 MDN profile for IMAP March 2003 11. Full Copyright Statement Copyright (C) The Internet Society (2003). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Melnikov Standards Track [Page 9] ================================================ FILE: docs/rfcs/rfc3516.IMAP4_Binary_content_extension.txt ================================================ Network Working Group L. Nerenberg Request for Comments: 3516 Orthanc Systems Category: Standards Track April 2003 IMAP4 Binary Content Extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2003). All Rights Reserved. Abstract This memo defines the Binary extension to the Internet Message Access Protocol (IMAP4). It provides a mechanism for IMAP4 clients and servers to exchange message body data without using a MIME content- transfer-encoding. 1. Conventions Used in this Document The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as described in [KEYWORD]. The abbreviation "CTE" means content-transfer-encoding. 2. Introduction The MIME extensions to Internet messaging allow for the transmission of non-textual (binary) message content [MIME-IMB]. Since the traditional transports for messaging are not always capable of passing binary data transparently, MIME provides encoding schemes that allow binary content to be transmitted over transports that are not otherwise able to do so. The overhead of MIME-encoding this content can be considerable in some contexts (e.g., slow radio links, streaming multimedia). Reducing the overhead associated with CTE schemes such as base64 Nerenberg Standards Track [Page 1] RFC 3516 IMAP4 Binary Content Extension April 2003 can give a noticeable reduction in resource consumption. The Binary extension lets the server perform CTE decoding prior to transmitting message data to the client. 3. Content-Transfer-Encoding Considerations Every IMAP4 body section has a MIME content-transfer-encoding. (Those without an explicit Content-Transfer-Encoding header are implicitly labeled as "7bit" content.) In the terminology of [MIME- IMB], the CTE specifies both a decoding algorithm and the domain of the decoded data. In this memo, "decoding" refers to the CTE decoding step described in [MIME-IMB]. Certain CTEs use an identity encoding transformation. For these CTEs there is no decoding required, however the domain of the underlying data may not be expressible in the IMAP4 protocol (e.g., MIME "binary" content containing NUL octets). To accommodate these cases the Binary extension introduces a new type of literal protocol element that is fully eight bit transparent. Thus, server processing of the FETCH BINARY command involves two logical steps: 1) perform any CTE-related decoding 2) determine the domain of the decoded data Step 2 is necessary to determine which protocol element should be used to transmit the decoded data. (See FETCH Response Extensions for further details.) 4. Framework for the IMAP4 Binary Extension This memo defines the following extensions to [IMAP4rev1]. 4.1. CAPABILITY Identification IMAP4 servers that support this extension MUST include "BINARY" in the response list to the CAPABILITY command. 4.2. FETCH Command Extensions This extension defines three new FETCH command data items. BINARY[] Requests that the specified section be transmitted after performing CTE-related decoding. Nerenberg Standards Track [Page 2] RFC 3516 IMAP4 Binary Content Extension April 2003 The argument, if present, requests that a subset of the data be returned. The semantics of a partial FETCH BINARY command are the same as for a partial FETCH BODY command, with the exception that the arguments refer to the DECODED section data. BINARY.PEEK[] An alternate form of FETCH BINARY that does not implicitly set the \Seen flag. BINARY.SIZE Requests the decoded size of the section (i.e., the size to expect in response to the corresponding FETCH BINARY request). Note: client authors are cautioned that this might be an expensive operation for some server implementations. Needlessly issuing this request could result in degraded performance due to servers having to calculate the value every time the request is issued. 4.3. FETCH Response Extensions This extension defines two new FETCH response data items. BINARY[<>] An or expressing the content of the specified section after removing any CTE-related encoding. If is present it refers to the offset within the DECODED section data. If the domain of the decoded data is "8bit" and the data does not contain the NUL octet, the server SHOULD return the data in a instead of a ; this allows the client to determine if the "8bit" data contains the NUL octet without having to explicitly scan the data stream for for NULs. If the server does not know how to decode the section's CTE, it MUST fail the request and issue a "NO" response that contains the "UNKNOWN-CTE" extended response code. Nerenberg Standards Track [Page 3] RFC 3516 IMAP4 Binary Content Extension April 2003 BINARY.SIZE The size of the section after removing any CTE-related encoding. The value returned MUST match the size of the or that will be returned by the corresponding FETCH BINARY request. If the server does not know how to decode the section's CTE, it MUST fail the request and issue a "NO" response that contains the "UNKNOWN-CTE" extended response code. 4.4. APPEND Command Extensions The APPEND command is extended to allow the client to append data containing NULs by using the syntax. The server MAY modify the CTE of the appended data, however any such transformation MUST NOT result in a loss of data. If the destination mailbox does not support the storage of binary content, the server MUST fail the request and issue a "NO" response that contains the "UNKNOWN-CTE" extended response code. 5. MIME Encoded Headers [MIME-MHE] defines an encoding that allows for non-US-ASCII text in message headers. This encoding is not the same as the content- transfer-encoding applied to message bodies, and the decoding transformations described in this memo do not apply to [MIME-MHE] encoded header text. A server MUST NOT perform any conversion of [MIME-MHE] encoded header text in response to any binary FETCH or APPEND request. 6. Implementation Considerations Messaging clients and servers have been notoriously lax in their adherence to the Internet CRLF convention for terminating lines of textual data in Internet protocols. When sending data using the Binary extension, servers MUST ensure that textual line-oriented sections are always transmitted using the IMAP4 CRLF line termination syntax, regardless of the underlying storage representation of the data on the server. A server may choose to store message body binary content in a non- encoded format. Regardless of the internal storage representation used, the server MUST issue BODYSTRUCTURE responses that describe the message as though the binary-encoded sections are encoded in a CTE Nerenberg Standards Track [Page 4] RFC 3516 IMAP4 Binary Content Extension April 2003 acceptable to the IMAP4 base specification. Furthermore, the results of a FETCH BODY MUST return the message body content in the format described by the corresponding FETCH BODYSTRUCTURE response. While the server is allowed to modify the CTE of APPENDed data, this should only be done when it is absolutely necessary. Gratuitous encoding changes will render useless most cryptographic operations that have been performed on the message. This extension provides an optimization that is useful in certain specific situations. It does not absolve clients from providing basic functionality (content transfer decoding) that should be available in all messaging clients. Clients supporting this extension SHOULD be prepared to perform their own CTE decoding operations. 7. Formal Protocol Syntax The following syntax specification uses the augmented Backus-Naur Form (ABNF) notation as used in [ABNF], and incorporates by reference the Core Rules defined in that document. This syntax augments the grammar specified in [IMAP4rev1]. append =/ "APPEND" SP mailbox [SP flag-list] [SP date-time] SP literal8 fetch-att =/ "BINARY" [".PEEK"] section-binary [partial] / "BINARY.SIZE" section-binary literal8 = "~{" number "}" CRLF *OCTET ; represents the number of OCTETs ; in the response string. msg-att-static =/ "BINARY" section-binary SP (nstring / literal8) / "BINARY.SIZE" section-binary SP number partial = "<" number "." nz-number ">" resp-text-code =/ "UNKNOWN-CTE" section-binary = "[" [section-part] "]" Nerenberg Standards Track [Page 5] RFC 3516 IMAP4 Binary Content Extension April 2003 8. Normative References [ABNF] Crocker, D., Editor, and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. [IMAP4rev1] Crispin, M., "Internet Message Access Protocol Version 4rev1", RFC 3501, March 2003. [KEYWORD] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MIME-IMB] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [MIME-MHE] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. 9. Security Considerations There are no known additional security issues with this extension beyond those described in the base protocol described in [IMAP4rev1]. 10. Intellectual Property The IETF takes no position regarding the validity or scope of any intellectual property or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; neither does it represent that it has made any effort to identify any such rights. Information on the IETF's procedures with respect to rights in standards-track and standards-related documentation can be found in BCP-11. Copies of claims of rights made available for publication and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementors or users of this specification can be obtained from the IETF Secretariat. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights which may cover technology that may be required to practice this standard. Please address the information to the IETF Executive Director. Nerenberg Standards Track [Page 6] RFC 3516 IMAP4 Binary Content Extension April 2003 11. Author's Address Lyndon Nerenberg Orthanc Systems 1606 - 10770 Winterburn Road Edmonton, Alberta Canada T5S 1T6 EMail: lyndon@orthanc.ab.ca Nerenberg Standards Track [Page 7] RFC 3516 IMAP4 Binary Content Extension April 2003 12. Full Copyright Statement Copyright (C) The Internet Society (2003). All Rights Reserved. This document and translations of it may be copied and furnished to others, and derivative works that comment on or otherwise explain it or assist in its implementation may be prepared, copied, published and distributed, in whole or in part, without restriction of any kind, provided that the above copyright notice and this paragraph are included on all such copies and derivative works. However, this document itself may not be modified in any way, such as by removing the copyright notice or references to the Internet Society or other Internet organizations, except as needed for the purpose of developing Internet standards in which case the procedures for copyrights defined in the Internet Standards process must be followed, or as required to translate it into languages other than English. The limited permissions granted above are perpetual and will not be revoked by the Internet Society or its successors or assigns. This document and the information contained herein is provided on an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Nerenberg Standards Track [Page 8] ================================================ FILE: docs/rfcs/rfc3691.IMAP_UNSELECT_command.txt ================================================ Network Working Group A. Melnikov Request for Comments: 3691 Isode Ltd. Category: Standards Track February 2004 Internet Message Access Protocol (IMAP) UNSELECT command Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2004). All Rights Reserved. Abstract This document defines an UNSELECT command that can be used to close the current mailbox in an Internet Message Access Protocol - version 4 (IMAP4) session without expunging it. Certain types of IMAP clients need to release resources associated with the selected mailbox without selecting a different mailbox. While IMAP4 provides this functionality (via a SELECT command with a nonexistent mailbox name or reselecting the same mailbox with EXAMINE command), a more clean solution is desirable. Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 2. UNSELECT command . . . . . . . . . . . . . . . . . . . . . . . 2 3. Security Considerations. . . . . . . . . . . . . . . . . . . . 3 4. Formal Syntax. . . . . . . . . . . . . . . . . . . . . . . . . 3 5. IANA Considerations. . . . . . . . . . . . . . . . . . . . . . 3 6. Acknowledgments. . . . . . . . . . . . . . . . . . . . . . . . 3 7. Normative References . . . . . . . . . . . . . . . . . . . . . 4 8. Author's Address . . . . . . . . . . . . . . . . . . . . . . . 4 9. Full Copyright Statement . . . . . . . . . . . . . . . . . . . 5 Melnikov Standards Track [Page 1] RFC 3691 IMAP UNSELECT command February 2004 1. Introduction Certain types of IMAP clients need to release resources associated with the selected mailbox without selecting a different mailbox. While [IMAP4] provides this functionality (via a SELECT command with a nonexistent mailbox name or reselecting the same mailbox with EXAMINE command), a more clean solution is desirable. [IMAP4] defines the CLOSE command that closes the selected mailbox as well as permanently removes all messages with the \Deleted flag set. However [IMAP4] lacks a command that simply closes the mailbox without expunging it. This document defines the UNSELECT command for this purpose. A server which supports this extension indicates this with a capability name of "UNSELECT". "C:" and "S:" in examples show lines sent by the client and server respectively. The keywords "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document when typed in uppercase are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 2. UNSELECT Command Arguments: none Responses: no specific responses for this command Result: OK - unselect completed, now in authenticated state BAD - no mailbox selected, or argument supplied but none permitted The UNSELECT command frees server's resources associated with the selected mailbox and returns the server to the authenticated state. This command performs the same actions as CLOSE, except that no messages are permanently removed from the currently selected mailbox. Example: C: A341 UNSELECT S: A341 OK Unselect completed Melnikov Standards Track [Page 2] RFC 3691 IMAP UNSELECT command February 2004 3. Security Considerations It is believed that this extension doesn't raise any additional security concerns not already discussed in [IMAP4]. 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. Non-terminals referenced but not defined below are as defined by [IMAP4]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. command-select /= "UNSELECT" 5. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/imap4-capabilities This document defines the UNSELECT IMAP capabilities. IANA has added this capability to the registry. 6. Acknowledgments UNSELECT command was originally implemented by Tim Showalter in Cyrus IMAP server. Also, the author of the document would like to thank Vladimir Butenko and Mark Crispin for reminding that UNSELECT has to be documented. Also thanks to Simon Josefsson for pointing out that there are multiple ways to implement UNSELECT. Melnikov Standards Track [Page 3] RFC 3691 IMAP UNSELECT command February 2004 7. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 2234, November 1997. 8. Author's Address Alexey Melnikov Isode Limited 5 Castle Business Village Hampton, Middlesex TW12 2BX EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Melnikov Standards Track [Page 4] RFC 3691 IMAP UNSELECT command February 2004 9. Full Copyright Statement Copyright (C) The Internet Society (2004). This document is subject to the rights, licenses and restrictions contained in BCP 78 and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Melnikov Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc4314.IMAP4_ACL_extension.txt ================================================ Network Working Group A. Melnikov Request for Comments: 4314 Isode Ltd. Obsoletes: 2086 December 2005 Category: Standards Track IMAP4 Access Control List (ACL) Extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2005). Abstract The Access Control List (ACL) extension (RFC 2086) of the Internet Message Access Protocol (IMAP) permits mailbox access control lists to be retrieved and manipulated through the IMAP protocol. This document is a revision of RFC 2086. It defines several new access control rights and clarifies which rights are required for different IMAP commands. Melnikov Standards Track [Page 1] RFC 4314 IMAP ACL December 2005 Table of Contents 1. Introduction and Overview .......................................3 1.1. Conventions Used in This Document ..........................3 2. Access Control ..................................................3 2.1. Standard Rights ............................................5 2.1.1. Obsolete Rights .....................................5 2.2. Rights Defined in RFC 2086 .................................8 3. Access control management commands and responses ................8 3.1. SETACL Command .............................................8 3.2. DELETEACL Command ..........................................9 3.3. GETACL Command ............................................10 3.4. LISTRIGHTS Command ........................................10 3.5. MYRIGHTS Command ..........................................11 3.6. ACL Response ..............................................11 3.7. LISTRIGHTS Response .......................................12 3.8. MYRIGHTS Response .........................................12 4. Rights Required to Perform Different IMAP4rev1 Commands ........12 5. Other Considerations ...........................................17 5.1. Additional Requirements and Implementation Notes ..........17 5.1.1. Servers ............................................17 5.1.2. Clients ............................................18 5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY Response Codes ............................................19 6. Security Considerations ........................................20 7. Formal Syntax ..................................................21 8. IANA Considerations ............................................22 9. Internationalization Considerations ............................22 Appendix A. Changes since RFC 2086 ................................23 Appendix B. Compatibility with RFC 2086 ...........................24 Appendix C. Known Deficiencies ....................................24 Appendix D. Acknowledgements ......................................25 Normative References ..............................................25 Informative References ............................................25 Melnikov Standards Track [Page 2] RFC 4314 IMAP ACL December 2005 1. Introduction and Overview The ACL (Access Control List) extension of the Internet Message Access Protocol [IMAP4] permits mailbox access control lists to be retrieved and manipulated through the IMAP protocol. This document is a revision of RFC 2086 [RFC2086]. It tries to clarify different ambiguities in RFC 2086, in particular, the use of UTF-8 [UTF-8] in access identifiers, which rights are required for different IMAP4 commands, and how READ-WRITE/READ-ONLY response codes are related to ACL. 1.1. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server respectively. In all examples "/" character is used as hierarchy separator. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [KEYWORDS]. The phrase "ACL server" is just a shortcut for saying "IMAP server that supports ACL extension as defined in this document". 2. Access Control The ACL extension is present in any IMAP4 implementation that returns "ACL" as one of the supported capabilities to the CAPABILITY command. A server implementation conformant to this document MUST also return rights (see below) not defined in Section 2.2 in the "RIGHTS=" capability. An access control list is a set of pairs. An ACL applies to a mailbox name. Access identifier (or just "identifier") is a UTF-8 [UTF-8] string. The identifier "anyone" is reserved to refer to the universal identity (all authentications, including anonymous). All user name strings accepted by the LOGIN or AUTHENTICATE commands to authenticate to the IMAP server are reserved as identifiers for the corresponding users. Identifiers starting with a dash ("-") are reserved for "negative rights", described below. All other identifier strings are interpreted in an implementation-defined manner. Melnikov Standards Track [Page 3] RFC 4314 IMAP ACL December 2005 Rights is a string listing a (possibly empty) set of alphanumeric characters, each character listing a set of operations that is being controlled. Lowercase letters are reserved for "standard" rights, listed in Section 2.1. (Note that for compatibility with deployed clients and servers uppercase rights are not allowed.) The set of standard rights can only be extended by a standards-track document. Digits are reserved for implementation- or site-defined rights. An implementation MAY tie rights together or MAY force rights to always or never be granted to particular identifiers. For example, in an implementation that uses UNIX mode bits, the rights "swite" are tied, the "a" right is always granted to the owner of a mailbox and is never granted to another user. If rights are tied in an implementation, the implementation must be conservative in granting rights in response to SETACL commands--unless all rights in a tied set are specified, none of that set should be included in the ACL entry for that identifier. A client can discover the set of rights that may be granted to a given identifier in the ACL for a given mailbox name by using the LISTRIGHTS command. It is possible for multiple identifiers in an access control list to apply to a given user. For example, an ACL may include rights to be granted to the identifier matching the user, one or more implementation-defined identifiers matching groups that include the user, and/or the identifier "anyone". How these rights are combined to determine the user's access is implementation defined. An implementation may choose, for example, to use the union of the rights granted to the applicable identifiers. An implementation may instead choose, for example, to use only those rights granted to the most specific identifier present in the ACL. A client can determine the set of rights granted to the logged-in user for a given mailbox name by using the MYRIGHTS command. When an identifier in an ACL starts with a dash ("-"), that indicates that associated rights are to be removed from the identifier prefixed by the dash. This is referred to as a "negative right". This differs from DELETEACL in that a negative right is added to the ACL and is a part of the calculation of the rights. Let's assume that an identifier "fred" refers to a user with login "fred". If the identifier "-fred" is granted the "w" right, that indicates that the "w" right is to be removed from users matching the identifier "fred", even though the user "fred" might have the "w" right as a consequence of some other identifier in the ACL. A DELETEACL of "fred" simply deletes the identifier "fred" from the ACL; it does not affect any rights that the user "fred" may get from another entry in the ACL, in particular it doesn't affect rights granted to the identifier "-fred". Melnikov Standards Track [Page 4] RFC 4314 IMAP ACL December 2005 Server implementations are not required to support "negative right" identifiers. 2.1. Standard Rights The currently defined standard rights are (note that the list below doesn't list all commands that use a particular right): l - lookup (mailbox is visible to LIST/LSUB commands, SUBSCRIBE mailbox) r - read (SELECT the mailbox, perform STATUS) s - keep seen/unseen information across sessions (set or clear \SEEN flag via STORE, also set \SEEN during APPEND/COPY/ FETCH BODY[...]) w - write (set or clear flags other than \SEEN and \DELETED via STORE, also set them during APPEND/COPY) i - insert (perform APPEND, COPY into mailbox) p - post (send mail to submission address for mailbox, not enforced by IMAP4 itself) k - create mailboxes (CREATE new sub-mailboxes in any implementation-defined hierarchy, parent mailbox for the new mailbox name in RENAME) x - delete mailbox (DELETE mailbox, old mailbox name in RENAME) t - delete messages (set or clear \DELETED flag via STORE, set \DELETED flag during APPEND/COPY) e - perform EXPUNGE and expunge as a part of CLOSE a - administer (perform SETACL/DELETEACL/GETACL/LISTRIGHTS) 2.1.1. Obsolete Rights Due to ambiguity in RFC 2086, some existing RFC 2086 server implementations use the "c" right to control the DELETE command. Others chose to use the "d" right to control the DELETE command. For the former group, let's define the "create" right as union of the "k" and "x" rights, and the "delete" right as union of the "e" and "t" rights. For the latter group, let's define the "create" rights as a synonym to the "k" right, and the "delete" right as union of the "e", "t", and "x" rights. For compatibility with RFC 2086, this section defines two virtual rights "d" and "c". If a client includes the "d" right in a rights list, then it MUST be treated as if the client had included every member of the "delete" right. (It is not an error for a client to specify both the "d" right and one or more members of the "delete" right, but the effect is no different than if just the "d" right or all members of the "delete" right had been specified.) Melnikov Standards Track [Page 5] RFC 4314 IMAP ACL December 2005 When any of the "delete" member rights is set in a list of rights, the server MUST also include the "d" right when returning the list in a MYRIGHTS or ACL response. This is to enable older clients conforming to RFC 2086 to work with newer servers. (*) Example: C: A001 SeTacl INBOX/Drafts David lrswida S: A001 OK Setacl complete The client has specified the "d" right in the SETACL command above and it expands to "et" on the server: C: A002 getacl INBOX/Drafts S: * ACL INBOX Fred rwipslxcetda David lrswideta S: A002 OK Getacl complete If the identifier specified in the LISTRIGHTS command can be granted any of the "delete" member rights on a mailbox, then the server MUST include the "d" right in the corresponding LISTRIGHTS response. (*) If the member rights aren't tied to non-member rights, then the "d" right is returned by itself in the LISTRIGHTS response. If any of the member rights needs to be tied to one (or more) non-member right, then the "d" right and all of the member rights need to be tied to the same non-member right(s) (**). If a client includes the "c" right in a rights list, then it MUST be treated as if the client had included every member of the "create" right. (It is not an error for a client to specify both the "c" right and one or more members of the "create" right, but the effect is no different than if just the "c" right or all members of the "create" right had been specified.) When any of the "create" member rights is set in a list of rights, the server MUST also include the "c" right when returning the list in a MYRIGHTS or ACL response. This is to enable older clients conforming to RFC 2086 to work with newer servers. (*) Example: C: A003 Setacl INBOX/Drafts Byron lrswikda S: A001 OK Setacl complete C: A002 getAcl INBOX/Drafts S: * ACL INBOX Fred rwipslxcetda Byron lrswikcdeta S: A002 OK Getacl complete The client has specified the "d" right in the SETACL command above and it expands to "et" on the server: As the client has specified the "k" right (which is a member of the "c" right), the server also returns the "c" right. Melnikov Standards Track [Page 6] RFC 4314 IMAP ACL December 2005 If the identifier specified in the LISTRIGHTS command can be granted any of the "create" member rights on a mailbox, then the server MUST include the "c" right in the corresponding LISTRIGHTS response. (*) If the member rights aren't tied to non-member rights, then the "c" right is returned by itself in the LISTRIGHTS response. If any of the member rights needs to be tied to one (or more) non-member right, then the "c" right and all of the member rights need to be tied to the same non-member right(s) (**). Example: The server that ties the rights as follows: lr s w i p k x t and c=k will return: S: * LISTRIGHTS archive/imap anyone "" lr s w i p k x t c d Example: The server that ties the rights as follows: lr s w i p k xte and c=k will return: S: * LISTRIGHTS archive/imap anyone "" lr s w i p k xte c d Example: The server that ties the rights as follows: lr s w i p k x te and c=k will return: S: * LISTRIGHTS archive/imap anyone "" lr s w i p k c x te d Example: The server that ties the rights as follows: lr swte i p k x and c=kx Melnikov Standards Track [Page 7] RFC 4314 IMAP ACL December 2005 will return: S: * LISTRIGHTS archive/imap anyone "" lr swted i p k x c (*) Clients conforming to this document MUST ignore the virtual "d" and "c" rights in MYRIGHTS, ACL, and LISTRIGHTS responses. (**) The IMAPEXT Working Group has debated this issue in great length and after reviewing existing ACL implementations concluded that this is a reasonable restriction. 2.2. Rights Defined in RFC 2086 The "RIGHTS=" capability MUST NOT include any of the rights defined in RFC 2086: "l", "r", "s", "w", "i", "p", "a", "c", "d", and the digits ("0" .. "9"). 3. Access control management commands and responses Servers, when processing a command that has an identifier as a parameter (i.e., any of SETACL, DELETEACL, and LISTRIGHTS commands), SHOULD first prepare the received identifier using "SASLprep" profile [SASLprep] of the "stringprep" algorithm [Stringprep]. If the preparation of the identifier fails or results in an empty string, the server MUST refuse to perform the command with a BAD response. Note that Section 6 recommends additional identifier's verification steps. 3.1. SETACL Command Arguments: mailbox name identifier access right modification Data: no specific data for this command Result: OK - setacl completed NO - setacl failure: can't set acl BAD - arguments invalid The SETACL command changes the access control list on the specified mailbox so that the specified identifier is granted permissions as specified in the third argument. The third argument is a string containing an optional plus ("+") or minus ("-") prefix, followed by zero or more rights characters. If the string starts with a plus, the following rights are added to any Melnikov Standards Track [Page 8] RFC 4314 IMAP ACL December 2005 existing rights for the identifier. If the string starts with a minus, the following rights are removed from any existing rights for the identifier. If the string does not start with a plus or minus, the rights replace any existing rights for the identifier. Note that an unrecognized right MUST cause the command to return the BAD response. In particular, the server MUST NOT silently ignore unrecognized rights. Example: C: A001 GETACL INBOX/Drafts S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswi S: A001 OK Getacl complete C: A002 SETACL INBOX/Drafts Chris +cda S: A002 OK Setacl complete C: A003 GETACL INBOX/Drafts S: * ACL INBOX/Drafts Fred rwipslxetad Chris lrswicdakxet S: A003 OK Getacl complete C: A035 SETACL INBOX/Drafts John lrQswicda S: A035 BAD Uppercase rights are not allowed C: A036 SETACL INBOX/Drafts John lrqswicda S: A036 BAD The q right is not supported 3.2. DELETEACL Command Arguments: mailbox name identifier Data: no specific data for this command Result: OK - deleteacl completed NO - deleteacl failure: can't delete acl BAD - arguments invalid The DELETEACL command removes any pair for the specified identifier from the access control list for the specified mailbox. Example: C: B001 getacl INBOX S: * ACL INBOX Fred rwipslxetad -Fred wetd $team w S: B001 OK Getacl complete C: B002 DeleteAcl INBOX Fred S: B002 OK Deleteacl complete Melnikov Standards Track [Page 9] RFC 4314 IMAP ACL December 2005 C: B003 GETACL INBOX S: * ACL INBOX -Fred wetd $team w S: B003 OK Getacl complete 3.3. GETACL Command Arguments: mailbox name Data: untagged responses: ACL Result: OK - getacl completed NO - getacl failure: can't get acl BAD - arguments invalid The GETACL command returns the access control list for mailbox in an untagged ACL response. Some implementations MAY permit multiple forms of an identifier to reference the same IMAP account. Usually, such implementations will have a canonical form that is stored internally. An ACL response caused by a GETACL command MAY include a canonicalized form of the identifier that might be different from the one used in the corresponding SETACL command. Example: C: A002 GETACL INBOX S: * ACL INBOX Fred rwipsldexta S: A002 OK Getacl complete 3.4. LISTRIGHTS Command Arguments: mailbox name identifier Data: untagged responses: LISTRIGHTS Result: OK - listrights completed NO - listrights failure: can't get rights list BAD - arguments invalid The LISTRIGHTS command takes a mailbox name and an identifier and returns information about what rights can be granted to the identifier in the ACL for the mailbox. Some implementations MAY permit multiple forms of an identifier to reference the same IMAP account. Usually, such implementations will have a canonical form that is stored internally. A LISTRIGHTS Melnikov Standards Track [Page 10] RFC 4314 IMAP ACL December 2005 response caused by a LISTRIGHTS command MUST always return the same form of an identifier as specified by the client. This is to allow the client to correlate the response with the command. Example: C: a001 LISTRIGHTS ~/Mail/saved smith S: * LISTRIGHTS ~/Mail/saved smith la r swicdkxte S: a001 OK Listrights completed Example: C: a005 listrights archive/imap anyone S: * LISTRIGHTS archive.imap anyone "" l r s w i p k x t e c d a 0 1 2 3 4 5 6 7 8 9 S: a005 Listrights successful 3.5. MYRIGHTS Command Arguments: mailbox name Data: untagged responses: MYRIGHTS Result: OK - myrights completed NO - myrights failure: can't get rights BAD - arguments invalid The MYRIGHTS command returns the set of rights that the user has to mailbox in an untagged MYRIGHTS reply. Example: C: A003 MYRIGHTS INBOX S: * MYRIGHTS INBOX rwiptsldaex S: A003 OK Myrights complete 3.6. ACL Response Data: mailbox name zero or more identifier rights pairs The ACL response occurs as a result of a GETACL command. The first string is the mailbox name for which this ACL applies. This is followed by zero or more pairs of strings; each pair contains the identifier for which the entry applies followed by the set of rights that the identifier has. Section 2.1.1 details additional server requirements related to handling of the virtual "d" and "c" rights. Melnikov Standards Track [Page 11] RFC 4314 IMAP ACL December 2005 3.7. LISTRIGHTS Response Data: mailbox name identifier required rights list of optional rights The LISTRIGHTS response occurs as a result of a LISTRIGHTS command. The first two strings are the mailbox name and identifier for which this rights list applies. Following the identifier is a string containing the (possibly empty) set of rights the identifier will always be granted in the mailbox. Following this are zero or more strings each containing a set of rights the identifier can be granted in the mailbox. Rights mentioned in the same string are tied together. The server MUST either grant all tied rights to the identifier in the mailbox or grant none. Section 2.1.1 details additional server requirements related to handling of the virtual "d" and "c" rights. The same right MUST NOT be listed more than once in the LISTRIGHTS command. 3.8. MYRIGHTS Response Data: mailbox name rights The MYRIGHTS response occurs as a result of a MYRIGHTS command. The first string is the mailbox name for which these rights apply. The second string is the set of rights that the client has. Section 2.1.1 details additional server requirements related to handling of the virtual "d" and "c" rights. 4. Rights Required to Perform Different IMAP4rev1 Commands Before executing a command, an ACL-compliant server MUST check which rights are required to perform it. This section groups command by functions they perform and list the rights required. It also gives the detailed description of any special processing required. For the purpose of this section the UID counterpart of a command is considered to be the same command, e.g., both UID COPY and COPY commands require the same set of rights. Melnikov Standards Track [Page 12] RFC 4314 IMAP ACL December 2005 The table below summarizes different rights or their combinations that are required in order to perform different IMAP operations. As it is not always possible to express complex right checking and interactions, the description after the table should be used as the primary reference. +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ |Operations\Rights | l | r | s | w | i | k | x | t | e | a |Any|Non| +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ | commands in authenticated state | +-------------------------------------------------------------------+ | LIST | + | | | | | | | | | | | | | SUBSCRIBE | * | | | | | | | | | | | * | | UNSUBSCRIBE | | | | | | | | | | | | + | | LSUB | * | | | | | | | | | | | * | |CREATE (for parent)| | | | | | + | | | | | | | | DELETE | | ? | | | | | + | ? | ? | | | | | RENAME | | | | | | + | + | | | | | | | SELECT/EXAMINE | | + | | | | | | | | | | | | STATUS | | + | | | | | | | | | | | | SETACL/DELETEACL | | | | | | | | | | + | | | | GETACL/LISTRIGHTS | | | | | | | | | | + | | | | MYRIGHTS | | | | | | | | | | | + | | | APPEND | | | ? | ? | + | | | ? | | | | | +-------------------------------------------------------------------+ | commands in selected state | +-------------------------------------------------------------------+ | COPY | | | ? | ? | + | | | ? | | | | | | EXPUNGE | | | | | | | | | + | | | | | CLOSE | | | | | | | | | ? | | | | | FETCH | | | ? | | | | | | | | | | | STORE flags | | | ? | ? | | | | ? | | | | | +-------------------+---+---+---+---+---+---+---+---+---+---+---+---+ Note: for all commands in the selected state, the "r" is implied, because it is required to SELECT/EXAMINE a mailbox. Servers are not required to check presence of the "r" right once a mailbox is successfully selected. Legend: + - The right is required * - Only one of the rights marked with * is required (see description below) ? - The right is OPTIONAL (see description below) "Any" - at least one of the "l", "r", "i", "k", "x", "a" rights is required "Non" - No rights required to perform the command Melnikov Standards Track [Page 13] RFC 4314 IMAP ACL December 2005 Listing and subscribing/unsubscribing mailboxes: LIST - "l" right is required. However, unlike other commands (e.g., SELECT) the server MUST NOT return a NO response if it can't list a mailbox. Note that if the user has "l" right to a mailbox "A/B", but not to its parent mailbox "A", the LIST command should behave as if the mailbox "A" doesn't exist, for example: C: A777 LIST "" * S: * LIST (\NoInferiors) "/" "A/B" S: * LIST () "/" "C" S: * LIST (\NoInferiors) "/" "C/D" S: A777 OK LIST completed SUBSCRIBE - "l" right is required only if the server checks for mailbox existence when performing SUBSCRIBE. UNSUBSCRIBE - no rights required to perform this operation. LSUB - "l" right is required only if the server checks for mailbox existence when performing SUBSCRIBE. However, unlike other commands (e.g., SELECT) the server MUST NOT return a NO response if it can't list a subscribed mailbox. Mailbox management: CREATE - "k" right on a nearest existing parent mailbox. When a new mailbox is created, it SHOULD inherit the ACL from the parent mailbox (if one exists) in the defined hierarchy. DELETE - "x" right on the mailbox. Note that some servers don't allow to delete a non-empty mailbox. If this is the case, the user would also need "r", "e", and "t" rights, in order to open the mailbox and empty it. The DELETE command MUST delete the ACL associated with the deleted mailbox. RENAME - Moving a mailbox from one parent to another requires the "x" right on the mailbox itself and the "k" right for the new parent. For example, if the user wants to rename the mailbox named "A/B/C" to "D/E", the user must have the "x" right for the mailbox "A/B/C" and the "k" right for the mailbox "D". The RENAME command SHOULD NOT change the ACLs on the renamed mailbox and submailboxes. Melnikov Standards Track [Page 14] RFC 4314 IMAP ACL December 2005 Copying or appending messages: Before performing a COPY/APPEND command, the server MUST check if the user has "i" right for the target mailbox. If the user doesn't have "i" right, the operation fails. Otherwise for each copied/appended message the server MUST check if the user has "t" right - when the message has \Deleted flag set "s" right - when the message has \Seen flag set "w" right - for all other message flags. Only when the user has a particular right are the corresponding flags stored for the newly created message. The server MUST NOT fail a COPY/APPEND if the user has no rights to set a particular flag. Example: C: A003 MYRIGHTS TargetMailbox S: * MYRIGHTS TargetMailbox rwis S: A003 OK Myrights complete C: A004 FETCH 1:3 (FLAGS) S: * 1 FETCH (FLAGS (\Draft \Deleted) S: * 2 FETCH (FLAGS (\Answered) S: * 3 FETCH (FLAGS ($Forwarded \Seen) S: A004 OK Fetch Completed C: A005 COPY 1:3 TargetMailbox S: A005 OK Copy completed C: A006 SELECT TargetMailbox ... S: A006 Select Completed Let's assume that the copied messages received message numbers 77:79. C: A007 FETCH 77:79 (FLAGS) S: * 77 FETCH (FLAGS (\Draft)) S: * 78 FETCH (FLAGS (\Answered)) S: * 79 FETCH (FLAGS ($Forwarded \Seen)) S: A007 OK Fetch Completed \Deleted flag was lost on COPY, as the user has no "t" right in the target mailbox. If the MYRIGHTS command with the tag A003 would have returned: S: * MYRIGHTS TargetMailbox rsti the response from the FETCH with the tag A007 would have been: C: A007 FETCH 77:79 (FLAGS) Melnikov Standards Track [Page 15] RFC 4314 IMAP ACL December 2005 S: * 77 FETCH (FLAGS (\Deleted)) S: * 78 FETCH (FLAGS ()) S: * 79 FETCH (FLAGS (\Seen)) S: A007 OK Fetch Completed In the latter case, \Answered, $Forwarded, and \Draft flags were lost on COPY, as the user has no "w" right in the target mailbox. Expunging the selected mailbox: EXPUNGE - "e" right on the selected mailbox. CLOSE - "e" right on the selected mailbox. If the server is unable to expunge the mailbox because the user doesn't have the "e" right, the server MUST ignore the expunge request, close the mailbox, and return the tagged OK response. Fetch information about a mailbox and its messages: SELECT/EXAMINE/STATUS - "r" right on the mailbox. FETCH - A FETCH request that implies setting \Seen flag MUST NOT set it, if the current user doesn't have "s" right. Changing flags: STORE - the server MUST check if the user has "t" right - when the user modifies \Deleted flag "s" right - when the user modifies \Seen flag "w" right - for all other message flags. STORE operation SHOULD NOT fail if the user has rights to modify at least one flag specified in the STORE, as the tagged NO response to a STORE command is not handled very well by deployed clients. Changing ACLs: SETACL/DELETEACL - "a" right on the mailbox. Reading ACLs: GETACL - "a" right on the mailbox. MYRIGHTS - any of the following rights is required to perform the operation: "l", "r", "i", "k", "x", "a". LISTRIGHTS - "a" right on the mailbox. Melnikov Standards Track [Page 16] RFC 4314 IMAP ACL December 2005 5. Other Considerations 5.1. Additional Requirements and Implementation Notes 5.1.1. Servers This document defines an additional capability that is used to announce the list of extra rights (excluding the ones defined in RFC 2086) supported by the server. The set of rights MUST include "t", "e", "x", and "k". Note that the extra rights can appear in any order. Example: C: 1 capability S: * CAPABILITY IMAP4REV1 STARTTLS LITERAL+ ACL RIGHTS=texk S: 1 OK completed Any server implementing an ACL extension MUST accurately reflect the current user's rights in FLAGS and PERMANENTFLAGS responses. Example: C: A142 SELECT INBOX S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L S: A142 OK [READ-WRITE] SELECT completed C: A143 MYRIGHTS INBOX S: * MYRIGHTS INBOX lrwis S: A143 OK completed Note that in order to get better performance the client MAY pipeline SELECT and MYRIGHTS commands: C: A142 SELECT INBOX C: A143 MYRIGHTS INBOX S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \*)] L S: A142 OK [READ-WRITE] SELECT completed S: * MYRIGHTS INBOX lrwis S: A143 OK completed Melnikov Standards Track [Page 17] RFC 4314 IMAP ACL December 2005 Servers MAY cache the rights a user has on a mailbox when the mailbox is selected, so that if a client's rights on a mailbox are changed with SETACL or DELETEACL, commands specific to the selected state (e.g., STORE, EXPUNGE) might not reflect the changed rights until the mailbox is re-selected. If the server checks the rights on each command, then it SHOULD send FLAGS and PERMANENTFLAGS responses if they have changed. If such server detects that the user no longer has read access to the mailbox, it MAY send an untagged BYE response and close connection. It MAY also refuse to execute all commands specific to the selected state until the mailbox is closed; however, server implementors should note that most clients don't handle NO responses very well. An ACL server MAY modify one or more ACLs for one or more identifiers as a side effect of modifying the ACL specified in a SETACL/DELETEACL. If the server does that, it MUST send untagged ACL response(s) to notify the client about the changes made. An ACL server implementation MUST treat received ACL modification commands as a possible ambiguity with respect to subsequent commands affected by the ACL, as described in Section 5.5 of [IMAP4]. Hence a pipeline SETACL + MYRIGHTS is an ambiguity with respect to the server, meaning that the server must execute the SETACL command to completion before the MYRIGHTS. However, clients are permitted to send such a pipeline. 5.1.2. Clients The following requirement is put on clients in order to allow for future extensibility. A client implementation that allows a user to read and update ACLs MUST preserve unrecognized rights that it doesn't allow the user to change. That is, if the client 1) can read ACLs and 2) can update ACLs but 3) doesn't allow the user to change the rights the client doesn't recognize, then it MUST preserve unrecognized rights. Otherwise the client could risk unintentionally removing permissions it doesn't understand. Melnikov Standards Track [Page 18] RFC 4314 IMAP ACL December 2005 5.2. Mapping of ACL Rights to READ-WRITE and READ-ONLY Response Codes A particular ACL server implementation MAY allow "shared multiuser access" to some mailboxes. "Shared multiuser access" to a mailbox means that multiple different users are able to access the same mailbox, if they have proper access rights. "Shared multiuser access" to the mailbox doesn't mean that the ACL for the mailbox is currently set to allow access by multiple users. Let's denote a "shared multiuser write access" as a "shared multiuser access" when a user can be granted flag modification rights (any of "w", "s", or "t"). Section 4 describes which rights are required for modifying different flags. If the ACL server implements some flags as shared for a mailbox (i.e., the ACL for the mailbox MAY be set up so that changes to those flags are visible to another user), let's call the set of rights associated with these flags (as described in Section 4) for that mailbox collectively as "shared flag rights". Note that the "shared flag rights" set MAY be different for different mailboxes. If the server doesn't support "shared multiuser write access" to a mailbox or doesn't implement shared flags on the mailbox, "shared flag rights" for the mailbox is defined to be the empty set. Example 1: Mailbox "banan" allows "shared multiuser write access" and implements flags \Deleted, \Answered, and $MDNSent as shared flags. "Shared flag rights" for the mailbox "banan" is a set containing flags "t" (because system flag \Deleted requires "t" right) and "w" (because both \Answered and $MDNSent require "w" right). Example 2: Mailbox "apple" allows "shared multiuser write access" and implements \Seen system flag as shared flag. "Shared flag rights" for the mailbox "apple" contains "s" right because system flag \Seen requires "s" right. Example 3: Mailbox "pear" allows "shared multiuser write access" and implements flags \Seen, \Draft as shared flags. "Shared flag rights" for the mailbox "apple" is a set containing flags "s" (because system flag \Seen requires "s" right) and "w" (because system flag \Draft requires "w" right). The server MUST include a READ-ONLY response code in the tagged OK response to a SELECT command if none of the following rights is granted to the current user: Melnikov Standards Track [Page 19] RFC 4314 IMAP ACL December 2005 "i", "e", and "shared flag rights"(***). The server SHOULD include a READ-WRITE response code in the tagged OK response if at least one of the "i", "e", or "shared flag rights"(***) is granted to the current user. (***) Note that a future extension to this document can extend the list of rights that causes the server to return the READ-WRITE response code. Example 1 (continued): The user that has "lrs" rights for the mailbox "banan". The server returns READ-ONLY response code on SELECT, as none of "iewt" rights is granted to the user. Example 2 (continued): The user that has "rit" rights for the mailbox "apple". The server returns READ-WRITE response code on SELECT, as the user has "i" right. Example 3 (continued): The user that has "rset" rights for the mailbox "pear". The server returns READ-WRITE response code on SELECT, as the user has "e" and "s" rights. 6. Security Considerations An implementation MUST make sure the ACL commands themselves do not give information about mailboxes with appropriately restricted ACLs. For example, when a user agent executes a GETACL command on a mailbox that the user has no permission to LIST, the server would respond to that request with the same error that would be used if the mailbox did not exist, thus revealing no existence information, much less the mailbox's ACL. IMAP clients implementing ACL that are able to modify ACLs SHOULD warn a user that wants to give full access (or even just the "a" right) to the special identifier "anyone". This document relies on [SASLprep] to describe steps required to perform identifier canonicalization (preparation). The preparation algorithm in SASLprep was specifically designed such that its output is canonical, and it is well-formed. However, due to an anomaly [PR29] in the specification of Unicode normalization, canonical equivalence is not guaranteed for a select few character sequences. Identifiers prepared with SASLprep can be stored and returned by an ACL server. The anomaly affects ACL manipulation and evaluation of identifiers containing the selected character sequences. These Melnikov Standards Track [Page 20] RFC 4314 IMAP ACL December 2005 sequences, however, do not appear in well-formed text. In order to address this problem, an ACL server MAY reject identifiers containing sequences described in [PR29] by sending the tagged BAD response. This is in addition to the requirement to reject identifiers that fail SASLprep preparation as described in Section 3. Other security considerations described in [IMAP4] are relevant to this document. In particular, ACL information is sent in the clear over the network unless confidentiality protection is negotiated. This can be accomplished either by the use of STARTTLS, negotiated privacy protection in the AUTHENTICATE command, or some other protection mechanism. 7. Formal Syntax Formal syntax is defined using ABNF [ABNF], extending the ABNF rules in Section 9 of [IMAP4]. Elements not defined here can be found in [ABNF] and [IMAP4]. Except as noted otherwise, all alphabetic characters are case insensitive. The use of uppercase or lowercase characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. LOWER-ALPHA = %x61-7A ;; a-z acl-data = "ACL" SP mailbox *(SP identifier SP rights) capability =/ rights-capa ;;capability is defined in [IMAP4] command-auth =/ setacl / deleteacl / getacl / listrights / myrights ;;command-auth is defined in [IMAP4] deleteacl = "DELETEACL" SP mailbox SP identifier getacl = "GETACL" SP mailbox identifier = astring listrights = "LISTRIGHTS" SP mailbox SP identifier listrights-data = "LISTRIGHTS" SP mailbox SP identifier SP rights *(SP rights) Melnikov Standards Track [Page 21] RFC 4314 IMAP ACL December 2005 mailbox-data =/ acl-data / listrights-data / myrights-data ;;mailbox-data is defined in [IMAP4] mod-rights = astring ;; +rights to add, -rights to remove ;; rights to replace myrights = "MYRIGHTS" SP mailbox myrights-data = "MYRIGHTS" SP mailbox SP rights new-rights = 1*LOWER-ALPHA ;; MUST include "t", "e", "x", and "k". ;; MUST NOT include standard rights listed ;; in section 2.2 rights = astring ;; only lowercase ASCII letters and digits ;; are allowed. rights-capa = "RIGHTS=" new-rights ;; RIGHTS=... capability setacl = "SETACL" SP mailbox SP identifier SP mod-rights 8. IANA Considerations IMAP4 capabilities are registered by publishing a standards-track or IESG-approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/imap4-capabilities This document defines the RIGHTS= IMAP capability. IANA has added this capability to the registry. 9. Internationalization Considerations Section 3 states requirements on servers regarding internationalization of identifiers. Melnikov Standards Track [Page 22] RFC 4314 IMAP ACL December 2005 Appendix A. Changes since RFC 2086 1. Changed the charset of "identifier" from US-ASCII to UTF-8. 2. Specified that mailbox deletion is controlled by the "x" right and EXPUNGE is controlled by the "e" right. 3. Added the "t" right that controls STORE \Deleted. Redefined the "d" right to be a macro for "e", "t", and possibly "x". 4. Added the "k" right that controls CREATE. Redefined the "c" right to be a macro for "k" and possibly "x". 5. Specified that the "a" right also controls DELETEACL. 6. Specified that the "r" right also controls STATUS. 7. Removed the requirement to check the "r" right for CHECK, SEARCH and FETCH, as this is required for SELECT/EXAMINE to be successful. 8. LISTRIGHTS requires the "a" right on the mailbox (same as SETACL). 9. Deleted "PARTIAL", this is a deprecated feature of RFC 1730. 10. Specified that the "w" right controls setting flags other than \Seen and \Deleted on APPEND. Also specified that the "s" right controls the \Seen flag and that the "t" right controls the \Deleted flag. 11. Specified that SUBSCRIBE is NOT allowed with the "r" right. 12. Specified that the "l" right controls SUBSCRIBE. 13. GETACL is NOT allowed with the "r" right, even though there are several implementations that allows that. If a user only has "r" right, GETACL can disclose information about identifiers existing on the mail system. 14. Clarified that RENAME requires the "k" right for the new parent and the "x" right for the old name. 15. Added new section that describes which rights are required and/or checked when performing various IMAP commands. 16. Added mail client security considerations when dealing with special identifier "anyone". 17. Clarified that negative rights are not the same as DELETEACL. 18. Added "Compatibility with RFC 2086" section. 19. Added section about mapping of ACL rights to READ-WRITE and READ-ONLY response codes. 20. Changed BNF to ABNF. 21. Added "Implementation Notes" section. 22. Updated "References" section. 23. Added more examples. 24. Clarified when the virtual "c" and "d" rights are returned in ACL, MYRIGHTS, and LISTRIGHTS responses. Melnikov Standards Track [Page 23] RFC 4314 IMAP ACL December 2005 Appendix B. Compatibility with RFC 2086 This non-normative section gives guidelines as to how an existing RFC 2086 server implementation may be updated to comply with this document. This document splits the "d" right into several new different rights: "t", "e", and possibly "x" (see Section 2.1.1 for more details). The "d" right remains for backward-compatibility, but it is a virtual right. There are two approaches for RFC 2086 server implementors to handle the "d" right and the new rights that have replaced it: a. Tie "t", "e" (and possibly "x) together - almost no changes. b. Implement separate "x", "t" and "e". Return the "d" right in a MYRIGHTS response or an ACL response containing ACL information when any of the "t", "e" (and "x") is granted. In a similar manner this document splits the "c" right into several new different rights: "k" and possibly "x" (see Section 2.1.1 for more details). The "c" right remains for backwards-compatibility but it is a virtual right. Again, RFC 2086 server implementors can choose to tie rights or to implement separate rights, as described above. Also check Sections 5.1.1 and 5.1.2, as well as Appendix A, to see other changes required. Server implementors should check which rights are required to invoke different IMAP4 commands as described in Section 4. Appendix C. Known Deficiencies This specification has some known deficiencies including: 1. This is inadequate to provide complete read-write access to mailboxes protected by Unix-style rights bits because there is no equivalent to "chown" and "chgrp" commands nor is there a good way to discover such limitations are present. 2. Because this extension leaves the specific semantics of how rights are combined by the server as implementation defined, the ability to build a user-friendly interface is limited. 3. Users, groups, and special identifiers (e.g., anyone) exist in the same namespace. The work-in-progress "ACL2" extension is intended to redesign this extension to address these deficiencies without the constraint of backward-compatibility and may eventually supercede this facility. Melnikov Standards Track [Page 24] RFC 4314 IMAP ACL December 2005 However, RFC 2086 is deployed in multiple implementations so this intermediate step, which fixes the straightforward deficiencies in a backward-compatible fashion, is considered worthwhile. Appendix D. Acknowledgements This document is a revision of RFC 2086 written by John G. Myers. Editor appreciates comments received from Mark Crispin, Chris Newman, Cyrus Daboo, John G. Myers, Dave Cridland, Ken Murchison, Steve Hole, Vladimir Butenko, Larry Greenfield, Robert Siemborski, Harrie Hazewinkel, Philip Guenther, Brian Candler, Curtis King, Lyndon Nerenberg, Lisa Dusseault, Arnt Gulbrandsen, and other participants of the IMAPEXT working group. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. [Stringprep] Hoffman, P. and M. Blanchet, "Preparation of Internationalized Strings ("stringprep")", RFC 3454, December 2002. [SASLprep] Zeilenga, K., "SASLprep: Stringprep Profile for User Names and Passwords", RFC 4013, February 2005. Informative References [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. [PR29] "Public Review Issue #29: Normalization Issue", February 2004, . Melnikov Standards Track [Page 25] RFC 4314 IMAP ACL December 2005 Author's Address Alexey Melnikov Isode Ltd. 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX GB EMail: alexey.melnikov@isode.com Melnikov Standards Track [Page 26] RFC 4314 IMAP ACL December 2005 Full Copyright Statement Copyright (C) The Internet Society (2005). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf- ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Melnikov Standards Track [Page 27] ================================================ FILE: docs/rfcs/rfc4315.IMAP_UIDPLUS_extension.txt ================================================ Network Working Group M. Crispin Request for Comments: 4315 December 2005 Obsoletes: 2359 Category: Standards Track Internet Message Access Protocol (IMAP) - UIDPLUS extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2005). Abstract The UIDPLUS extension of the Internet Message Access Protocol (IMAP) provides a set of features intended to reduce the amount of time and resources used by some client operations. The features in UIDPLUS are primarily intended for disconnected-use clients. 1. Introduction and Overview The UIDPLUS extension is present in any IMAP server implementation that returns "UIDPLUS" as one of the supported capabilities to the CAPABILITY command. The UIDPLUS extension defines an additional command. In addition, this document recommends new status response codes in IMAP that SHOULD be returned by all server implementations, regardless of whether or not the UIDPLUS extension is implemented. The added facilities of the features in UIDPLUS are optimizations; clients can provide equivalent functionality, albeit less efficiently, by using facilities in the base protocol. 1.1. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. Crispin Standards Track [Page 1] RFC 4315 IMAP - UIDPLUS Extension December 2005 The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. A "UID set" is similar to the [IMAP] sequence set; however, the "*" value for a sequence number is not permitted. 2. Additional Commands The following command definition is an extension to [IMAP] section 6.4. 2.1. UID EXPUNGE Command Arguments: sequence set Data: untagged responses: EXPUNGE Result: OK - expunge completed NO - expunge failure (e.g., permission denied) BAD - command unknown or arguments invalid The UID EXPUNGE command permanently removes all messages that both have the \Deleted flag set and have a UID that is included in the specified sequence set from the currently selected mailbox. If a message either does not have the \Deleted flag set or has a UID that is not included in the specified sequence set, it is not affected. This command is particularly useful for disconnected use clients. By using UID EXPUNGE instead of EXPUNGE when resynchronizing with the server, the client can ensure that it does not inadvertantly remove any messages that have been marked as \Deleted by other clients between the time that the client was last connected and the time the client resynchronizes. If the server does not support the UIDPLUS capability, the client should fall back to using the STORE command to temporarily remove the \Deleted flag from messages it does not want to remove, then issuing the EXPUNGE command. Finally, the client should use the STORE command to restore the \Deleted flag on the messages in which it was temporarily removed. Alternatively, the client may fall back to using just the EXPUNGE command, risking the unintended removal of some messages. Crispin Standards Track [Page 2] RFC 4315 IMAP - UIDPLUS Extension December 2005 Example: C: A003 UID EXPUNGE 3000:3002 S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: A003 OK UID EXPUNGE completed 3. Additional Response Codes The following response codes are extensions to the response codes defined in [IMAP] section 7.1. With limited exceptions, discussed below, server implementations that advertise the UIDPLUS extension SHOULD return these response codes. In the case of a mailbox that has permissions set so that the client can COPY or APPEND to the mailbox, but not SELECT or EXAMINE it, the server SHOULD NOT send an APPENDUID or COPYUID response code as it would disclose information about the mailbox. In the case of a mailbox that has UIDNOTSTICKY status (as defined below), the server MAY omit the APPENDUID or COPYUID response code as it is not meaningful. If the server does not return the APPENDUID or COPYUID response codes, the client can discover this information by selecting the destination mailbox. The location of messages placed in the destination mailbox by COPY or APPEND can be determined by using FETCH and/or SEARCH commands (e.g., for Message-ID or some unique marker placed in the message in an APPEND). APPENDUID Followed by the UIDVALIDITY of the destination mailbox and the UID assigned to the appended message in the destination mailbox, indicates that the message has been appended to the destination mailbox with that UID. If the server also supports the [MULTIAPPEND] extension, and if multiple messages were appended in the APPEND command, then the second value is a UID set containing the UIDs assigned to the appended messages, in the order they were transmitted in the APPEND command. This UID set may not contain extraneous UIDs or the symbol "*". Note: the UID set form of the APPENDUID response code MUST NOT be used if only a single message was appended. In particular, a server MUST NOT send a range such as 123:123. This is because a client that does not support [MULTIAPPEND] expects only a single UID and not a UID set. Crispin Standards Track [Page 3] RFC 4315 IMAP - UIDPLUS Extension December 2005 UIDs are assigned in strictly ascending order in the mailbox (refer to [IMAP], section 2.3.1.1) and UID ranges are as in [IMAP]; in particular, note that a range of 12:10 is exactly equivalent to 10:12 and refers to the sequence 10,11,12. This response code is returned in a tagged OK response to the APPEND command. COPYUID Followed by the UIDVALIDITY of the destination mailbox, a UID set containing the UIDs of the message(s) in the source mailbox that were copied to the destination mailbox and containing the UIDs assigned to the copied message(s) in the destination mailbox, indicates that the message(s) have been copied to the destination mailbox with the stated UID(s). The source UID set is in the order the message(s) were copied; the destination UID set corresponds to the source UID set and is in the same order. Neither of the UID sets may contain extraneous UIDs or the symbol "*". UIDs are assigned in strictly ascending order in the mailbox (refer to [IMAP], section 2.3.1.1) and UID ranges are as in [IMAP]; in particular, note that a range of 12:10 is exactly equivalent to 10:12 and refers to the sequence 10,11,12. This response code is returned in a tagged OK response to the COPY command. UIDNOTSTICKY The selected mailbox is supported by a mail store that does not support persistent UIDs; that is, UIDVALIDITY will be different each time the mailbox is selected. Consequently, APPEND or COPY to this mailbox will not return an APPENDUID or COPYUID response code. This response code is returned in an untagged NO response to the SELECT command. Note: servers SHOULD NOT have any UIDNOTSTICKY mail stores. This facility exists to support legacy mail stores in which it is technically infeasible to support persistent UIDs. This should be avoided when designing new mail stores. Crispin Standards Track [Page 4] RFC 4315 IMAP - UIDPLUS Extension December 2005 Example: C: A003 APPEND saved-messages (\Seen) {297} C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@example.com C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK [APPENDUID 38505 3955] APPEND completed C: A004 COPY 2:4 meeting S: A004 OK [COPYUID 38505 304,319:320 3956:3958] Done C: A005 UID COPY 305:310 meeting S: A005 OK No matching messages, so nothing copied C: A006 COPY 2 funny S: A006 OK Done C: A007 SELECT funny S: * 1 EXISTS S: * 1 RECENT S: * OK [UNSEEN 1] Message 1 is first unseen S: * OK [UIDVALIDITY 3857529045] Validity session-only S: * OK [UIDNEXT 2] Predicted next UID S: * NO [UIDNOTSTICKY] Non-persistent UIDs S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen)] Limited S: A007 OK [READ-WRITE] SELECT completed In this example, A003 and A004 demonstrate successful appending and copying to a mailbox that returns the UIDs assigned to the messages. A005 is an example in which no messages were copied; this is because in A003, we see that message 2 had UID 304, and message 3 had UID 319; therefore, UIDs 305 through 310 do not exist (refer to section 2.3.1.1 of [IMAP] for further explanation). A006 is an example of a message being copied that did not return a COPYUID; and, as expected, A007 shows that the mail store containing that mailbox does not support persistent UIDs. 4. Formal Syntax Formal syntax is defined using ABNF [ABNF], which extends the ABNF rules defined in [IMAP]. The IMAP4 ABNF should be imported before attempting to validate these rules. append-uid = uniqueid capability =/ "UIDPLUS" Crispin Standards Track [Page 5] RFC 4315 IMAP - UIDPLUS Extension December 2005 command-select =/ uid-expunge resp-code-apnd = "APPENDUID" SP nz-number SP append-uid resp-code-copy = "COPYUID" SP nz-number SP uid-set SP uid-set resp-text-code =/ resp-code-apnd / resp-code-copy / "UIDNOTSTICKY" ; incorporated before the expansion rule of ; atom [SP 1*] ; that appears in [IMAP] uid-expunge = "UID" SP "EXPUNGE" SP sequence-set uid-set = (uniqueid / uid-range) *("," uid-set) uid-range = (uniqueid ":" uniqueid) ; two uniqueid values and all values ; between these two regards of order. ; Example: 2:4 and 4:2 are equivalent. Servers that support [MULTIAPPEND] will have the following extension to the above rules: append-uid =/ uid-set ; only permitted if client uses [MULTIAPPEND] ; to append multiple messages. 5. Security Considerations The COPYUID and APPENDUID response codes return information about the mailbox, which may be considered sensitive if the mailbox has permissions set that permit the client to COPY or APPEND to the mailbox, but not SELECT or EXAMINE it. Consequently, these response codes SHOULD NOT be issued if the client does not have access to SELECT or EXAMINE the mailbox. 6. IANA Considerations This document constitutes registration of the UIDPLUS capability in the imap4-capabilities registry, replacing [RFC2359]. 7. Normative References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. Crispin Standards Track [Page 6] RFC 4315 IMAP - UIDPLUS Extension December 2005 [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension", RFC 3502, March 2003. 8. Informative References [RFC2359] Myers, J., "IMAP4 UIDPLUS extension", RFC 2359, June 1998. 9. Changes from RFC 2359 This document obsoletes [RFC2359]. However, it is based upon that document, and takes substantial text from it (albeit with numerous clarifications in wording). [RFC2359] implied that a server must always return COPYUID/APPENDUID data; thus suggesting that in such cases the server should return arbitrary data if the destination mailbox did not support persistent UIDs. This document adds the UIDNOTSTICKY response code to indicate that a mailbox does not support persistent UIDs, and stipulates that a UIDPLUS server does not return COPYUID/APPENDUID data when the COPY (or APPEND) destination mailbox has UIDNOTSTICKY status. Author's Address Mark R. Crispin Networks and Distributed Computing University of Washington 4545 15th Avenue NE Seattle, WA 98105-4527 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin Standards Track [Page 7] RFC 4315 IMAP - UIDPLUS Extension December 2005 Full Copyright Statement Copyright (C) The Internet Society (2005). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf- ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Crispin Standards Track [Page 8] ================================================ FILE: docs/rfcs/rfc4466.Collected_extensions_to_IMAP4_ABNF.txt ================================================ Network Working Group A. Melnikov Request for Comments: 4466 Isode Ltd. Updates: 2088, 2342, 3501, 3502, 3516 C. Daboo Category: Standards Track April 2006 Collected Extensions to IMAP4 ABNF Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2006). Abstract Over the years, many documents from IMAPEXT and LEMONADE working groups, as well as many individual documents, have added syntactic extensions to many base IMAP commands described in RFC 3501. For ease of reference, this document collects most of such ABNF changes in one place. This document also suggests a set of standard patterns for adding options and extensions to several existing IMAP commands defined in RFC 3501. The patterns provide for compatibility between existing and future extensions. This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. It also includes part of the errata to RFC 3501. This document doesn't specify any semantic changes to the listed RFCs. Melnikov & Daboo Standards Track [Page 1] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 Table of Contents 1. Introduction ....................................................2 1.1. Purpose of This Document ...................................2 1.2. Conventions Used in This Document ..........................3 2. IMAP ABNF Extensions ............................................3 2.1. Optional Parameters with the SELECT/EXAMINE Commands .......3 2.2. Extended CREATE Command ....................................4 2.3. Extended RENAME Command ....................................5 2.4. Extensions to FETCH and UID FETCH Commands .................6 2.5. Extensions to STORE and UID STORE Commands .................6 2.6. Extensions to SEARCH Command ...............................7 2.6.1. Extended SEARCH Command .............................7 2.6.2. ESEARCH untagged response ...........................8 2.7. Extensions to APPEND Command ...............................8 3. Formal Syntax ...................................................9 4. Security Considerations ........................................14 5. Normative References ...........................................15 6. Acknowledgements ...............................................15 1. Introduction 1.1. Purpose of This Document This document serves several purposes: 1. rationalize and generalize ABNF for some existing IMAP extensions; 2. collect the ABNF in one place in order to minimize cross references between documents; 3. define building blocks for future extensions so that they can be used together in a compatible way. It is expected that a future revision of this document will be incorporated into a revision of RFC 3501. This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. It also includes part of the errata to RFC 3501. This document doesn't specify any semantic changes to the listed RFCs. The ABNF in section 6 of RFC 2342 got rewritten to conform to the ABNF syntax as defined in RFC 4234 and to reference new non-terminals from RFC 3501. It was also restructured to allow for better readability. There were no changes "on the wire". Section 2 extends ABNF for SELECT, EXAMINE, CREATE, RENAME, FETCH/UID FETCH, STORE/UID STORE, SEARCH, and APPEND commands in a consistent manner. Extensions to all the commands but APPEND have the same Melnikov & Daboo Standards Track [Page 2] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 structure. Extensibility for the APPEND command was done slightly differently in order to preserve backward compatibility with existing extensions. Section 2 also defines a new ESEARCH response, whose purpose is to define a better version of the SEARCH response defined in RFC 3501. Section 3 defines the collected ABNF that replaces pieces of ABNF in the aforementioned RFCs. The collected ABNF got generalized to allow for easier future extensibility. 1.2. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [KEYWORDS]. 2. IMAP ABNF Extensions This section is not normative. It provides some background on the intended use of different extensions and it gives some guidance about how future extensions should extend the described commands. 2.1. Optional Parameters with the SELECT/EXAMINE Commands This document adds the ability to include one or more parameters with the IMAP SELECT (section 6.3.1 of [IMAP4]) or EXAMINE (section 6.3.2 of [IMAP4]) commands, to turn on or off certain standard behaviors, or to add new optional behaviors required for a particular extension. There are two possible modes of operation: o A global state change where a single use of the optional parameter will affect the session state from that time on, irrespective of subsequent SELECT/EXAMINE commands. o A per-mailbox state change that will affect the session only for the duration of the new selected state. A subsequent SELECT/EXAMINE without the optional parameter will cancel its effect for the newly selected mailbox. Optional parameters to the SELECT or EXAMINE commands are added as a parenthesized list of attribute/value pairs, and appear after the mailbox name in the standard SELECT or EXAMINE command. The order of individual parameters is arbitrary. A parameter value is optional Melnikov & Daboo Standards Track [Page 3] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 and may consist of atoms, strings, or lists in a specific order. If the parameter value is present, it always appears in parentheses (*). Any parameter not defined by extensions that the server supports must be rejected with a BAD response. Example: C: a SELECT INBOX (ANNOTATE) S: ... S: a OK SELECT complete In the above example, a single parameter is used with the SELECT command. Example: C: a EXAMINE INBOX (ANNOTATE RESPONSES ("UID Responses") CONDSTORE) S: ... S: a OK EXAMINE complete In the above example, three parameters are used with the EXAMINE command. The second parameter consists of two items: an atom "RESPONSES" followed by a quoted string. Example: C: a SELECT INBOX (BLURDYBLOOP) S: a BAD Unknown parameter in SELECT command In the above example, a parameter not supported by the server is used. This results in the BAD response from the server. (*) - if a parameter has a mandatory value, which can always be represented as a number or a sequence-set, the parameter value does not need the enclosing (). See ABNF for more details. 2.2. Extended CREATE Command Arguments: mailbox name OPTIONAL list of CREATE parameters Responses: no specific responses for this command Result: OK - create completed NO - create failure: cannot create mailbox with that name BAD - argument(s) invalid Melnikov & Daboo Standards Track [Page 4] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 This document adds the ability to include one or more parameters with the IMAP CREATE command (see section 6.3.3 of [IMAP4]), to turn on or off certain standard behaviors, or to add new optional behaviors required for a particular extension. No CREATE parameters are defined in this document. Optional parameters to the CREATE command are added as a parenthesized list of attribute/value pairs after the mailbox name. The order of individual parameters is arbitrary. A parameter value is optional and may consist of atoms, strings, or lists in a specific order. If the parameter value is present, it always appears in parentheses (*). Any parameter not defined by extensions that the server supports must be rejected with a BAD response. (*) - if a parameter has a mandatory value, which can always be represented as a number or a sequence-set, the parameter value does not need the enclosing (). See ABNF for more details. 2.3. Extended RENAME Command Arguments: existing mailbox name new mailbox name OPTIONAL list of RENAME parameters Responses: no specific responses for this command Result: OK - rename completed NO - rename failure: cannot rename mailbox with that name, cannot rename to mailbox with that name, etc. BAD - argument(s) invalid This document adds the ability to include one or more parameters with the IMAP RENAME command (see section 6.3.5 of [IMAP4]), to turn on or off certain standard behaviors, or to add new optional behaviors required for a particular extension. No RENAME parameters are defined in this document. Optional parameters to the RENAME command are added as a parenthesized list of attribute/value pairs after the new mailbox name. The order of individual parameters is arbitrary. A parameter value is optional and may consist of atoms, strings, or lists in a specific order. If the parameter value is present, it always appears in parentheses (*). Any parameter not defined by extensions that the server supports must be rejected with a BAD response. Melnikov & Daboo Standards Track [Page 5] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 (*) - if a parameter has a mandatory value, which can always be represented as a number or a sequence-set, the parameter value does not need the enclosing (). See ABNF for more details. 2.4. Extensions to FETCH and UID FETCH Commands Arguments: sequence set message data item names or macro OPTIONAL fetch modifiers Responses: untagged responses: FETCH Result: OK - fetch completed NO - fetch error: cannot fetch that data BAD - command unknown or arguments invalid This document extends the syntax of the FETCH and UID FETCH commands (see section 6.4.5 of [IMAP4]) to include optional FETCH modifiers. No fetch modifiers are defined in this document. The order of individual modifiers is arbitrary. Each modifier is an attribute/value pair. A modifier value is optional and may consist of atoms and/or strings and/or lists in a specific order. If the modifier value is present, it always appears in parentheses (*). Any modifiers not defined by extensions that the server supports must be rejected with a BAD response. (*) - if a modifier has a mandatory value, which can always be represented as a number or a sequence-set, the modifier value does not need the enclosing (). See ABNF for more details. 2.5. Extensions to STORE and UID STORE Commands Arguments: message set OPTIONAL store modifiers message data item name value for message data item Responses: untagged responses: FETCH Result: OK - store completed NO - store error: cannot store that data BAD - command unknown or arguments invalid This document extends the syntax of the STORE and UID STORE commands (see section 6.4.6 of [IMAP4]) to include optional STORE modifiers. No store modifiers are defined in this document. Melnikov & Daboo Standards Track [Page 6] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 The order of individual modifiers is arbitrary. Each modifier is an attribute/value pair. A modifier value is optional and may consist of atoms and/or strings and/or lists in a specific order. If the modifier value is present, it always appears in parentheses (*). Any modifiers not defined by extensions that the server supports must be rejected with a BAD response. (*) - if a modifier has a mandatory value, which can always be represented as a number or a sequence-set, the modifier value does not need the enclosing (). See ABNF for more details. 2.6. Extensions to SEARCH Command 2.6.1. Extended SEARCH Command Arguments: OPTIONAL result specifier OPTIONAL [CHARSET] specification searching criteria (one or more) Responses: REQUIRED untagged response: SEARCH (*) Result: OK - search completed NO - search error: cannot search that [CHARSET] or criteria BAD - command unknown or arguments invalid This section updates definition of the SEARCH command described in section 6.4.4 of [IMAP4]. The SEARCH command is extended to allow for result options. This document does not define any result options. The order of individual options is arbitrary. Individual options may contain parameters enclosed in parentheses (**). If an option has parameters, they consist of atoms and/or strings and/or lists in a specific order. Any options not defined by extensions that the server supports must be rejected with a BAD response. (*) - An extension to the SEARCH command may require another untagged response, or no untagged response to be returned. Section 2.6.2 defines a new ESEARCH untagged response that replaces the SEARCH untagged response. Note that for a given extended SEARCH command the SEARCH and ESEARCH responses SHOULD be mutually exclusive, i.e., only one of them should be returned. (**) - if an option has a mandatory parameter, which can always be represented as a number or a sequence-set, the option parameter does not need the enclosing (). See ABNF for more details. Melnikov & Daboo Standards Track [Page 7] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 2.6.2. ESEARCH untagged response Contents: one or more search-return-data pairs The ESEARCH response SHOULD be sent as a result of an extended SEARCH or UID SEARCH command specified in section 2.6.1. The ESEARCH response starts with an optional search correlator. If it is missing, then the response was not caused by a particular IMAP command, whereas if it is present, it contains the tag of the command that caused the response to be returned. The search correlator is followed by an optional UID indicator. If this indicator is present, all data in the ESEARCH response refers to UIDs, otherwise all returned data refers to message numbers. The rest of the ESEARCH response contains one or more search data pairs. Each pair starts with unique return item name, followed by a space and the corresponding data. Search data pairs may be returned in any order. Unless specified otherwise by an extension, any return item name SHOULD appear only once in an ESEARCH response. Example: S: * ESEARCH UID COUNT 5 ALL 4:19,21,28 Example: S: * ESEARCH (TAG "a567") UID COUNT 5 ALL 4:19,21,28 Example: S: * ESEARCH COUNT 5 ALL 1:17,21 2.7. Extensions to APPEND Command The IMAP BINARY extension [BINARY] extends the APPEND command to allow a client to append data containing NULs by using the syntax. The ABNF was rewritten to allow for easier extensibility by IMAP extensions. This document hasn't specified any semantical changes to the [BINARY] extension. In addition, the non-terminal "literal8" defined in [BINARY] got extended to allow for non-synchronizing literals if both [BINARY] and [LITERAL+] extensions are supported by the server. The IMAP MULTIAPPEND extension [MULTIAPPEND] extends the APPEND command to allow a client to append multiple messages atomically. This document defines a common syntax for the APPEND command that takes into consideration syntactic extensions defined by both [BINARY] and [MULTIAPPEND] extensions. Melnikov & Daboo Standards Track [Page 8] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 3. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. Non-terminals referenced but not defined below are as defined by [IMAP4]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of uppercase or lowercase characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. append = "APPEND" SP mailbox 1*append-message ;; only a single append-message may appear ;; if MULTIAPPEND [MULTIAPPEND] capability ;; is not present append-message = append-opts SP append-data append-ext = append-ext-name SP append-ext-value ;; This non-terminal define extensions to ;; to message metadata. append-ext-name = tagged-ext-label append-ext-value= tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. append-data = literal / literal8 / append-data-ext append-data-ext = tagged-ext ;; This non-terminal shows recommended syntax ;; for future extensions, ;; i.e., a mandatory label followed ;; by parameters. append-opts = [SP flag-list] [SP date-time] *(SP append-ext) ;; message metadata charset = atom / quoted ;; Exact syntax is defined in [CHARSET]. create = "CREATE" SP mailbox [create-params] ;; Use of INBOX gives a NO error. Melnikov & Daboo Standards Track [Page 9] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 create-params = SP "(" create-param *( SP create-param) ")" create-param-name = tagged-ext-label create-param = create-param-name [SP create-param-value] create-param-value= tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. esearch-response = "ESEARCH" [search-correlator] [SP "UID"] *(SP search-return-data) ;; Note that SEARCH and ESEARCH responses ;; SHOULD be mutually exclusive, ;; i.e., only one of the response types ;; should be ;; returned as a result of a command. examine = "EXAMINE" SP mailbox [select-params] ;; modifies the original IMAP EXAMINE command ;; to accept optional parameters fetch = "FETCH" SP sequence-set SP ("ALL" / "FULL" / "FAST" / fetch-att / "(" fetch-att *(SP fetch-att) ")") [fetch-modifiers] ;; modifies the original IMAP4 FETCH command to ;; accept optional modifiers fetch-modifiers = SP "(" fetch-modifier *(SP fetch-modifier) ")" fetch-modifier = fetch-modifier-name [ SP fetch-modif-params ] fetch-modif-params = tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. fetch-modifier-name = tagged-ext-label literal8 = "~{" number ["+"] "}" CRLF *OCTET ;; A string that might contain NULs. ;; represents the number of OCTETs ;; in the response string. ;; The "+" is only allowed when both LITERAL+ and ;; BINARY extensions are supported by the server. Melnikov & Daboo Standards Track [Page 10] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 mailbox-data =/ Namespace-Response / esearch-response Namespace = nil / "(" 1*Namespace-Descr ")" Namespace-Command = "NAMESPACE" Namespace-Descr = "(" string SP (DQUOTE QUOTED-CHAR DQUOTE / nil) *(Namespace-Response-Extension) ")" Namespace-Response-Extension = SP string SP "(" string *(SP string) ")" Namespace-Response = "NAMESPACE" SP Namespace SP Namespace SP Namespace ;; This response is currently only allowed ;; if the IMAP server supports [NAMESPACE]. ;; The first Namespace is the Personal Namespace(s) ;; The second Namespace is the Other Users' Namespace(s) ;; The third Namespace is the Shared Namespace(s) rename = "RENAME" SP mailbox SP mailbox [rename-params] ;; Use of INBOX as a destination gives ;; a NO error, unless rename-params ;; is not empty. rename-params = SP "(" rename-param *( SP rename-param) ")" rename-param = rename-param-name [SP rename-param-value] rename-param-name = tagged-ext-label rename-param-value= tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. response-data = "*" SP response-payload CRLF response-payload= resp-cond-state / resp-cond-bye / mailbox-data / message-data / capability-data search = "SEARCH" [search-return-opts] SP search-program search-correlator = SP "(" "TAG" SP tag-string ")" Melnikov & Daboo Standards Track [Page 11] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 search-program = ["CHARSET" SP charset SP] search-key *(SP search-key) ;; CHARSET argument to SEARCH MUST be ;; registered with IANA. search-return-data = search-modifier-name SP search-return-value ;; Note that not every SEARCH return option ;; is required to have the corresponding ;; ESEARCH return data. search-return-opts = SP "RETURN" SP "(" [search-return-opt *(SP search-return-opt)] ")" search-return-opt = search-modifier-name [SP search-mod-params] search-return-value = tagged-ext-val ;; Data for the returned search option. ;; A single "nz-number"/"number" value ;; can be returned as an atom (i.e., without ;; quoting). A sequence-set can be returned ;; as an atom as well. search-modifier-name = tagged-ext-label search-mod-params = tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. select = "SELECT" SP mailbox [select-params] ;; modifies the original IMAP SELECT command to ;; accept optional parameters select-params = SP "(" select-param *(SP select-param) ")" select-param = select-param-name [SP select-param-value] ;; a parameter to SELECT may contain one or ;; more atoms and/or strings and/or lists. select-param-name= tagged-ext-label select-param-value= tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. status-att-list = status-att-val *(SP status-att-val) ;; Redefines status-att-list from RFC 3501. Melnikov & Daboo Standards Track [Page 12] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 ;; status-att-val is defined in RFC 3501 errata status-att-val = ("MESSAGES" SP number) / ("RECENT" SP number) / ("UIDNEXT" SP nz-number) / ("UIDVALIDITY" SP nz-number) / ("UNSEEN" SP number) ;; Extensions to the STATUS responses ;; should extend this production. ;; Extensions should use the generic ;; syntax defined by tagged-ext. store = "STORE" SP sequence-set [store-modifiers] SP store-att-flags ;; extend [IMAP4] STORE command syntax ;; to allow for optional store-modifiers store-modifiers = SP "(" store-modifier *(SP store-modifier) ")" store-modifier = store-modifier-name [SP store-modif-params] store-modif-params = tagged-ext-val ;; This non-terminal shows recommended syntax ;; for future extensions. store-modifier-name = tagged-ext-label tag-string = string ;; tag of the command that caused ;; the ESEARCH response, sent as ;; a string. tagged-ext = tagged-ext-label SP tagged-ext-val ;; recommended overarching syntax for ;; extensions tagged-ext-label = tagged-label-fchar *tagged-label-char ;; Is a valid RFC 3501 "atom". tagged-label-fchar = ALPHA / "-" / "_" / "." tagged-label-char = tagged-label-fchar / DIGIT / ":" Melnikov & Daboo Standards Track [Page 13] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 tagged-ext-comp = astring / tagged-ext-comp *(SP tagged-ext-comp) / "(" tagged-ext-comp ")" ;; Extensions that follow this general ;; syntax should use nstring instead of ;; astring when appropriate in the context ;; of the extension. ;; Note that a message set or a "number" ;; can always be represented as an "atom". ;; An URL should be represented as ;; a "quoted" string. tagged-ext-simple = sequence-set / number tagged-ext-val = tagged-ext-simple / "(" [tagged-ext-comp] ")" 4. Security Considerations This document updates ABNF in RFCs 2088, 2342, 3501, 3502, and 3516. The updated documents must be consulted for security considerations for the extensions that they define. As a protocol gets more complex, parser bugs become more common including buffer overflow, denial of service, and other common security coding errors. To the extent that this document makes the parser more complex, it makes this situation worse. To the extent that this document makes the parser more consistent and thus simpler, the situation is improved. The impact will depend on how many deployed IMAP extensions are consistent with this document. Implementers are encouraged to take care of these issues when extending existing implementations. Future IMAP extensions should strive for consistency and simplicity to the greatest extent possible. Extensions to IMAP commands that are permitted in NOT AUTHENTICATED state are more sensitive to these security issues due to the larger possible attacker community prior to authentication, and the fact that some IMAP servers run with elevated privileges in that state. This document does not extend any commands permitted in NOT AUTHENTICATED state. Future IMAP extensions to commands permitted in NOT AUTHENTICATED state should favor simplicity over consistency or extensibility. Melnikov & Daboo Standards Track [Page 14] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 5. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [CHARSET] Freed, N. and J. Postel, "IANA Charset Registration Procedures", BCP 19, RFC 2978, October 2000. [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension", RFC 3502, March 2003. [NAMESPACE] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, May 1998. [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, January 1997. [BINARY] Nerenberg, L., "IMAP4 Binary Content Extension", RFC 3516, April 2003. 6. Acknowledgements This documents is based on ideas proposed by Pete Resnick, Mark Crispin, Ken Murchison, Philip Guenther, Randall Gellens, and Lyndon Nerenberg. However, all errors and omissions must be attributed to the authors of the document. Thanks to Philip Guenther, Dave Cridland, Mark Crispin, Chris Newman, Elwyn Davies, and Barry Leiba for comments and corrections. literal8 syntax was taken from RFC 3516. Melnikov & Daboo Standards Track [Page 15] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 Authors' Addresses Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex, TW12 2BX UK EMail: Alexey.Melnikov@isode.com Cyrus Daboo EMail: cyrus@daboo.name Melnikov & Daboo Standards Track [Page 16] RFC 4466 Collected Extensions to IMAP4 ABNF April 2006 Full Copyright Statement Copyright (C) The Internet Society (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is provided by the IETF Administrative Support Activity (IASA). Melnikov & Daboo Standards Track [Page 17] ================================================ FILE: docs/rfcs/rfc4467.IMAP_URLAUTH_extension.txt ================================================ Network Working Group M. Crispin Request for Comments: 4467 University of Washington Updates: 3501 May 2006 Category: Standards Track Internet Message Access Protocol (IMAP) - URLAUTH Extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2006). Abstract This document describes the URLAUTH extension to the Internet Message Access Protocol (IMAP) (RFC 3501) and the IMAP URL Scheme (IMAPURL) (RFC 2192). This extension provides a means by which an IMAP client can use URLs carrying authorization to access limited message data on the IMAP server. An IMAP server that supports this extension indicates this with a capability name of "URLAUTH". 1. Introduction In [IMAPURL], a URL of the form imap://fred@example.com/INBOX/;uid=20 requires authorization as userid "fred". However, [IMAPURL] implies that it only supports authentication and confuses the concepts of authentication and authorization. The URLAUTH extension defines an authorization mechanism for IMAP URLs to replace [IMAPURL]'s authentication-only mechanism. URLAUTH conveys authorization in the URL string itself and reuses a portion of the syntax of the [IMAPURL] authentication mechanism to convey the authorization identity (which also defines the default namespace in [IMAP]). The URLAUTH extension provides a means by which an authorized user of an IMAP server can create URLAUTH-authorized IMAP URLs. A URLAUTH- authorized URL conveys authorization (not authentication) to the data Crispin Standards Track [Page 1] RFC 4467 IMAP - URLAUTH Extension May 2006 addressed by that URL. This URL can be used in another IMAP session to access specific content on the IMAP server, without otherwise providing authorization to any other data (such as other data in the mailbox specified in the URL) owned by the authorizing user. Conceptually, a URLAUTH-authorized URL can be thought of as a "pawn ticket" that carries no authentication information and can be redeemed by whomever presents it. However, unlike a pawn ticket, URLAUTH has optional mechanisms to restrict the usage of a URLAUTH- authorized URL. Using these mechanisms, URLAUTH-authorized URLs can be usable by: . anonymous (the "pawn ticket" model) . authenticated users only . a specific authenticated user only . message submission acting on behalf of a specific user only There is also a mechanism for expiration. A URLAUTH-authorized URL can be used in the argument to the BURL command in message composition, as described in [BURL], for such purposes as allowing a client (with limited memory or other resources) to submit a message forward or to resend from an IMAP mailbox without requiring the client to fetch that message data. The URLAUTH is generated using an authorization mechanism name and an authorization token, which is generated using a secret mailbox access key. An IMAP client can request that the server generate and assign a new mailbox access key (thus effectively revoking all current URLs using URLAUTH with the old mailbox access key) but cannot set the mailbox access key to a key of its own choosing. 1.1. Conventions Used in this Document The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in [KEYWORDS]. The formal syntax uses the Augmented Backus-Naur Form (ABNF) notation including the core rules defined in Appendix A of [ABNF]. In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. If a single "C:" or "S:" label applies to multiple lines, then the line breaks between those lines are for editorial clarity only and are not part of the actual protocol exchange. Crispin Standards Track [Page 2] RFC 4467 IMAP - URLAUTH Extension May 2006 2. Concepts 2.1. URLAUTH The URLAUTH is a component, appended at the end of a URL, that conveys authorization to access the data addressed by that URL. It contains an authorized access identifier, an authorization mechanism name, and an authorization token. The authorization token is generated from the URL, the authorized access identifier, the authorization mechanism name, and a mailbox access key. 2.2. Mailbox Access Key The mailbox access key is a random string with at least 128 bits of entropy. It is generated by software (not by the human user) and MUST be unpredictable. Each user has a table of mailboxes and an associated mailbox access key for each mailbox. Consequently, the mailbox access key is per- user and per-mailbox. In other words, two users sharing the same mailbox each have a different mailbox access key for that mailbox, and each mailbox accessed by a single user also has a different mailbox access key. 2.3. Authorized Access Identifier The authorized access identifier restricts use of the URLAUTH authorized URL to certain users authorized on the server, as described in section 3. 2.4. Authorization Mechanism The authorization mechanism is the algorithm by which the URLAUTH is generated and subsequently verified, using the mailbox access key. 2.4.1. INTERNAL Authorization Mechanism This specification defines the INTERNAL mechanism, which uses a token generation algorithm of the server's choosing and does not involve disclosure of the mailbox access key to the client. Note: The token generation algorithm chosen by the server implementation should be modern and reasonably secure. At the time of the writing of this document, an [HMAC] such as HMAC-SHA1 is recommended. Crispin Standards Track [Page 3] RFC 4467 IMAP - URLAUTH Extension May 2006 If it becomes necessary to change the token generation algorithm of the INTERNAL mechanism (e.g., because an attack against the current algorithm has been discovered), all currently existing URLAUTH-authorized URLs are invalidated by the change in algorithm. Since this would be an unpleasant surprise to applications that depend upon the validity of a URLAUTH-authorized URL, and there is no good way to do a bulk update of existing deployed URLs, it is best to avoid this situation by using a secure algorithm as opposed to one that is "good enough". Server implementations SHOULD consider the possibility of changing the algorithm. In some cases, it may be desirable to implement the change of algorithm in a way that newly-generated tokens use the new algorithm, but that for a limited period of time tokens using either the new or old algorithm can be validated. Consequently, the server SHOULD incorporate some means of identifying the token generation algorithm within the token. Although this specification is extensible for other mechanisms, none are defined in this document. In addition to the mechanism name itself, other mechanisms may have mechanism-specific data, which is to be interpreted according to the definition of that mechanism. 2.5. Authorization Token The authorization token is a deterministic string of at least 128 bits that an entity with knowledge of the secret mailbox access key and URL authorization mechanism can use to verify the URL. 3. IMAP URL Extensions [IMAPURL] is extended by allowing the addition of ";EXPIRE=" and ";URLAUTH=::" to IMAP URLs that refer to a specific message or message parts. The URLAUTH is comprised of ";URLAUTH=::" and MUST be at the end of the URL. URLAUTH does not apply to, and MUST NOT be used with, any IMAP URL that refers to an entire IMAP server, a list of mailboxes, an entire IMAP mailbox, or IMAP search results. When ";EXPIRE=" is used, this indicates the latest date and time that the URL is valid. After that date and time, the URL has expired, and server implementations MUST reject the URL. If ";EXPIRE=" is not used, the URL has no expiration, but still can be revoked as discussed below. Crispin Standards Track [Page 4] RFC 4467 IMAP - URLAUTH Extension May 2006 The URLAUTH takes the form ";URLAUTH=::". It is composed of three parts. The portion provides the authorized access identifiers, which may constrain the operations and users that are permitted to use this URL. The portion provides the authorization mechanism used by the IMAP server to generate the authorization token that follows. The portion provides the authorization token. The "submit+" access identifier prefix, followed by a userid, indicates that only a userid authorized as a message submission entity on behalf of the specified userid is permitted to use this URL. The IMAP server does not validate the specified userid but does validate that the IMAP session has an authorization identity that is authorized as a message submission entity. The authorized message submission entity MUST validate the userid prior to contacting the IMAP server. The "user+" access identifier prefix, followed by a userid, indicates that use of this URL is limited to IMAP sessions that are logged in as the specified userid (that is, have authorization identity as that userid). Note: If a SASL mechanism that provides both authorization and authentication identifiers is used to authenticate to the IMAP server, the "user+" access identifier MUST match the authorization identifier. The "authuser" access identifier indicates that use of this URL is limited to IMAP sessions that are logged in as an authorized user (that is, have authorization identity as an authorized user) of that IMAP server. Use of this URL is prohibited to anonymous IMAP sessions. The "anonymous" access identifier indicates that use of this URL is not restricted by session authorization identity; that is, any IMAP session in authenticated or selected state (as defined in [IMAP]), including anonymous sessions, may issue a URLFETCH using this URL. The authorization token is represented as an ASCII-encoded hexadecimal string, which is used to authorize the URL. The length and the calculation of the authorization token depends upon the mechanism used; but, in all cases, the authorization token is at least 128 bits (and therefore at least 32 hexadecimal digits). Crispin Standards Track [Page 5] RFC 4467 IMAP - URLAUTH Extension May 2006 4. Discussion of URLAUTH Authorization Issues In [IMAPURL], the userid before the "@" in the URL has two purposes: 1) It provides context for user-specific mailbox paths such as "INBOX". 2) It specifies that resolution of the URL requires logging in as that user and limits use of that URL to only that user. An obvious limitation of using the same field for both purposes is that the URL can only be resolved by the mailbox owner. URLAUTH overrides the second purpose of the userid in the IMAP URL and by default permits the URL to be resolved by any user permitted by the access identifier. The "user+" access identifier limits resolution of that URL to a particular userid, whereas the "submit+" access identifier is more general and simply requires that the session be authorized by a user that has been granted a "submit" role within the authentication system. Use of either of these access identifiers makes it impossible for an attacker, spying on the session, to use the same URL, either directly or by submission to a message submission entity. The "authuser" and "anonymous" access identifiers do not have this level of protection and should be used with caution. These access identifiers are primarily useful for public export of data from an IMAP server, without requiring that it be copied to a web or anonymous FTP server. Refer to the Security Considerations for more details. 5. Generation of URLAUTH-Authorized URLs A URLAUTH-authorized URL is generated from an initial URL as follows: An initial URL is built, ending with ";URLAUTH=" but without the "::" components. An authorization mechanism is selected and used to calculate the authorization token, with the initial URL as the data and a secret known to the IMAP server as the key. The URLAUTH-authorized URL is generated by taking the initial URL and appending ":", the URL authorization mechanism name, ":", and the ASCII-encoded hexadecimal representation of the authorization token. Crispin Standards Track [Page 6] RFC 4467 IMAP - URLAUTH Extension May 2006 Note: ASCII-encoded hexadecimal is used instead of BASE64 because a BASE64 representation may have "=" padding characters, which would be problematic in a URL. In the INTERNAL mechanism, the mailbox access key for that mailbox is the secret known to the IMAP server, and a server-selected algorithm is used as described in section 2.4.1. 6. Validation of URLAUTH-authorized URLs A URLAUTH-authorized URL is validated as follows: The URL is split at the ":" that separates "" from ":" in the ";URLAUTH=::" portion of the URL. The ":" portion is first parsed and saved as the authorization mechanism and the authorization token. The URL is truncated, discarding the ":" described above, to create a "rump URL" (the URL minus the ":" and the ":" portion). The rump URL is then analyzed to identify the mailbox. If the mailbox cannot be identified, an authorization token is calculated on the rump URL, using random "plausible" keys (selected by the server) as needed, before returning a validation failure. This prevents timing attacks aimed at identifying mailbox names. If the mailbox can be identified, the authorization token is calculated on the rump URL and a secret known to the IMAP server using the given URL authorization mechanism. Validation is successful if, and only if, the calculated authorization token for that mechanism matches the authorization token supplied in ";URLAUTH=::". Removal of the "::" portion of the URL MUST be the only operation applied to the URLAUTH-authorized URL to get the rump URL. In particular, URL percent escape decoding and case-folding (including to the domain part of the URL) MUST NOT occur. In the INTERNAL mechanism, the mailbox access key for that mailbox is used as the secret known to the IMAP server, and the same server- selected algorithm used for generating URLs is used to calculate the authorization token for verification. Crispin Standards Track [Page 7] RFC 4467 IMAP - URLAUTH Extension May 2006 7. Additional Commands These commands are extensions to the [IMAP] base protocol. The section headings of these commands are intended to correspond with where they would be located in the base protocol document if they were part of that document. BASE.6.3.RESETKEY. RESETKEY Command Arguments: optional mailbox name optional mechanism name(s) Responses: none other than in result Result: OK - RESETKEY completed, URLMECH containing new data NO - RESETKEY error: can't change key of that mailbox BAD - command unknown or arguments invalid The RESETKEY command has two forms. The first form accepts a mailbox name as an argument and generates a new mailbox access key for the given mailbox in the user's mailbox access key table, replacing any previous mailbox access key (and revoking any URLs that were authorized with a URLAUTH using that key) in that table. By default, the mailbox access key is generated for the INTERNAL mechanism; other mechanisms can be specified with the optional mechanism argument. The second form, with no arguments, removes all mailbox access keys in the user's mailbox access key table, revoking all URLs currently authorized using URLAUTH by the user. Any current IMAP session logged in as the user that has the mailbox selected will receive an untagged OK response with the URLMECH status response code (see section BASE.7.1.URLMECH for more details about the URLMECH status response code). Example: C: a31 RESETKEY S: a31 OK All keys removed C: a32 RESETKEY INBOX S: a32 OK [URLMECH INTERNAL] mechs C: a33 RESETKEY INBOX XSAMPLE S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done Crispin Standards Track [Page 8] RFC 4467 IMAP - URLAUTH Extension May 2006 BASE.6.3.GENURLAUTH. GENURLAUTH Command Argument: one or more URL/mechanism pairs Response: untagged response: GENURLAUTH Result: OK - GENURLAUTH completed NO - GENURLAUTH error: can't generate a URLAUTH BAD - command unknown or arguments invalid The GENURLAUTH command requests that the server generate a URLAUTH- authorized URL for each of the given URLs using the given URL authorization mechanism. The server MUST validate each supplied URL as follows: (1) The mailbox component of the URL MUST refer to an existing mailbox. (2) The server component of the URL MUST contain a valid userid that identifies the owner of the mailbox access key table that will be used to generate the URLAUTH-authorized URL. As a consequence, the iserver rule of [IMAPURL] is modified so that iuserauth is mandatory. Note: the server component of the URL is generally the logged in userid and server. If not, then the logged in userid and server MUST have owner-type access to the mailbox access key table owned by the userid and server indicated by the server component of the URL. (3) There is a valid access identifier that, in the case of "submit+" and "user+", will contain a valid userid. This userid is not necessarily the same as the owner userid described in (2). (4) The server MAY also verify that the iuid and/or isection components (if present) are valid. If any of the above checks fail, the server MUST return a tagged BAD response with the following exception. If an invalid userid is supplied as the mailbox access key owner and/or as part of the access identifier, the server MAY issue a tagged OK response with a generated mailbox key that always fails validation when used with a URLFETCH command. This exception prevents an attacker from validating userids. Crispin Standards Track [Page 9] RFC 4467 IMAP - URLAUTH Extension May 2006 If there is currently no mailbox access key for the given mailbox in the owner's mailbox access key table, one is automatically generated. That is, it is not necessary to use RESETKEY prior to first-time use of GENURLAUTH. If the command is successful, a GENURLAUTH response code is returned listing the requested URLs as URLAUTH-authorized URLs. Examples: C: a775 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ ;section=1.2" INTERNAL S: a775 BAD missing access identifier in supplied URL C: a776 GENURLAUTH "imap://example.com/Shared/;uid=20/ ;section=1.2;urlauth=submit+fred" INTERNAL S: a776 BAD missing owner username in supplied URL C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ ;section=1.2;urlauth=submit+fred" INTERNAL S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" S: a777 OK GENURLAUTH completed BASE.6.3.URLFETCH. URLFETCH Command Argument: one or more URLs Response: untagged response: URLFETCH Result: OK - urlfetch completed NO - urlfetch failed due to server internal error BAD - command unknown or arguments invalid The URLFETCH command requests that the server return the text data associated with the specified IMAP URLs, as described in [IMAPURL] and extended by this document. The data is returned for all validated URLs, regardless of whether or not the session would otherwise be able to access the mailbox containing that data via SELECT or EXAMINE. Note: This command does not require that the URL refer to the selected mailbox; nor does it require that any mailbox be selected. It also does not in any way interfere with any selected mailbox. Crispin Standards Track [Page 10] RFC 4467 IMAP - URLAUTH Extension May 2006 The URLFETCH command effectively executes with the access of the userid in the server component of the URL (which is generally the userid that issued the GENURLAUTH). By itself, the URLAUTH does NOT grant access to the data; once validated, it grants whatever access to the data is held by the userid in the server component of the URL. That access may have changed since the GENURLAUTH was done. The URLFETCH command MUST return an untagged URLFETCH response and a tagged OK response to any URLFETCH command that is syntactically valid. A NO response indicates a server internal failure that may be resolved on later retry. Note: The possibility of a NO response is to accommodate implementations that would otherwise have to issue an untagged BYE with a fatal error due to an inability to respond to a valid request. In an ideal world, a server SHOULD NOT issue a NO response. The server MUST return NIL for any IMAP URL that references an entire IMAP server, a list of mailboxes, an entire IMAP mailbox, or IMAP search results. Example: Note: For clarity, this example uses the LOGIN command, which SHOULD NOT be used over a non-encrypted communication path. This example is of a submit server, obtaining a message segment for a message that it has already validated was submitted by "fred". S: * OK [CAPABILITY IMAP4REV1 URLAUTH] example.com IMAP server C: a001 LOGIN submitserver secret S: a001 OK submitserver logged in C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ ;section=1.2;urlauth=submit+fred:internal :91354a473744909de610943775f92038" S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 ;urlauth=submit+fred:internal :91354a473744909de610943775f92038" {28} S: Si vis pacem, para bellum. S: S: a002 OK URLFETCH completed Crispin Standards Track [Page 11] RFC 4467 IMAP - URLAUTH Extension May 2006 8. Additional Responses These responses are extensions to the [IMAP] base protocol. The section headings of these responses are intended to correspond with where they would be located in the base protocol document if they were part of that document. BASE.7.1.URLMECH. URLMECH Status Response Code The URLMECH status response code is followed by a list of URL authorization mechanism names. Mechanism names other than INTERNAL may be appended with an "=" and BASE64-encoded form of mechanism- specific data. This status response code is returned in an untagged OK response in response to a RESETKEY, SELECT, or EXAMINE command. In the case of the RESETKEY command, this status response code can be sent in the tagged OK response instead of requiring a separate untagged OK response. Example: C: a33 RESETKEY INBOX XSAMPLE S: a33 OK [URLMECH INTERNAL XSAMPLE=P34OKhO7VEkCbsiYY8rGEg==] done In this example, the server supports the INTERNAL mechanism and an experimental mechanism called XSAMPLE, which also holds some mechanism-specific data (the name "XSAMPLE" is for illustrative purposes only). BASE.7.4.GENURLAUTH. GENURLAUTH Response Contents: One or more URLs The GENURLAUTH response returns the URLAUTH-authorized URL(s) requested by a GENURLAUTH command. Example: C: a777 GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/ ;section=1.2;urlauth=submit+fred" INTERNAL S: * GENURLAUTH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 ;urlauth=submit+fred:internal:91354a473744909de610943775f92038" S: a777 OK GENURLAUTH completed Crispin Standards Track [Page 12] RFC 4467 IMAP - URLAUTH Extension May 2006 BASE.7.4.URLFETCH. URLFETCH Response Contents: One or more URL/nstring pairs The URLFETCH response returns the message text data associated with one or more IMAP URLs, as described in [IMAPURL] and extended by this document. This response occurs as the result of a URLFETCH command. The returned data string is NIL if the URL is invalid for any reason (including validation failure). If the URL is valid, but the IMAP fetch of the body part returned NIL (this should not happen), the returned data string should be the empty string ("") and not NIL. Note: This command does not require that the URL refer to the selected mailbox; nor does it require that any mailbox be selected. It also does not in any way interfere with any selected mailbox. Example: C: a002 URLFETCH "imap://joe@example.com/INBOX/;uid=20/ ;section=1.2;urlauth=submit+fred:internal :91354a473744909de610943775f92038" S: * URLFETCH "imap://joe@example.com/INBOX/;uid=20/;section=1.2 ;urlauth=submit+fred:internal :91354a473744909de610943775f92038" {28} S: Si vis pacem, para bellum. S: S: a002 OK URLFETCH completed 9. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. The following modifications are made to the Formal Syntax in [IMAP]: resetkey = "RESETKEY" [SP mailbox *(SP mechanism)] capability =/ "URLAUTH" command-auth =/ resetkey / genurlauth / urlfetch resp-text-code =/ "URLMECH" SP "INTERNAL" *(SP mechanism ["=" base64]) genurlauth = "GENURLAUTH" 1*(SP url-rump SP mechanism) genurlauth-data = "*" SP "GENURLAUTH" 1*(SP url-full) Crispin Standards Track [Page 13] RFC 4467 IMAP - URLAUTH Extension May 2006 url-full = astring ; contains authimapurlfull as defined below url-rump = astring ; contains authimapurlrump as defined below urlfetch = "URLFETCH" 1*(SP url-full) urlfetch-data = "*" SP "URLFETCH" 1*(SP url-full SP nstring) The following extensions are made to the Formal Syntax in [IMAPURL]: authimapurl = "imap://" enc-user [iauth] "@" hostport "/" imessagepart ; replaces "imapurl" and "iserver" rules for ; URLAUTH authorized URLs authimapurlfull = authimapurl iurlauth authimapurlrump = authimapurl iurlauth-rump enc-urlauth = 32*HEXDIG enc-user = 1*achar ; same as "enc_user" in RFC 2192 iurlauth = iurlauth-rump ":" mechanism ":" enc-urlauth iurlauth-rump = [expire] ";URLAUTH=" access access = ("submit+" enc-user) / ("user+" enc-user) / "authuser" / "anonymous" expire = ";EXPIRE=" date-time ; date-time defined in [DATETIME] mechanism = "INTERNAL" / 1*(ALPHA / DIGIT / "-" / ".") ; case-insensitive ; new mechanisms MUST be registered with IANA Crispin Standards Track [Page 14] RFC 4467 IMAP - URLAUTH Extension May 2006 10. Security Considerations Security considerations are discussed throughout this memo. The mailbox access key SHOULD have at least 128 bits of entropy (refer to [RANDOM] for more details) and MUST be unpredictable. The server implementation of the INTERNAL mechanism SHOULD consider the possibility of needing to change the token generation algorithm, and SHOULD incorporate some means of identifying the token generation algorithm within the token. The URLMECH status response code may expose sensitive data in the mechanism-specific data for mechanisms other than INTERNAL. A server implementation MUST implement a configuration that will not return a URLMECH status response code unless some mechanism is provided that protects the session from snooping, such as a TLS or SASL security layer that provides confidentiality protection. The calculation of an authorization token with a "plausible" key if the mailbox can not be identified is necessary to avoid attacks in which the server is probed to see if a particular mailbox exists on the server by measuring the amount of time taken to reject a known bad name versus some other name. To protect against a computational denial-of-service attack, a server MAY impose progressively longer delays on multiple URL requests that fail validation. The decision to use the "authuser" access identifier should be made with caution. An "authuser" access identifier can be used by any authorized user of the IMAP server; therefore, use of this access identifier should be limited to content that may be disclosed to any authorized user of the IMAP server. The decision to use the "anonymous" access identifier should be made with extreme caution. An "anonymous" access identifier can be used by anyone; therefore, use of this access identifier should be limited to content that may be disclosed to anyone. Many IMAP servers do not permit anonymous access; in this case, the "anonymous" access identifier is equivalent to "authuser", but this MUST NOT be relied upon. Although this specification does not prohibit the theoretical capability to generate a URL with a server component other than the logged in userid and server, this capability should only be provided Crispin Standards Track [Page 15] RFC 4467 IMAP - URLAUTH Extension May 2006 when the logged in userid/server has been authorized as equivalent to the server component userid/server, or otherwise has access to that userid/server mailbox access key table. 11. IANA Considerations This document constitutes registration of the URLAUTH capability in the imap4-capabilities registry. URLAUTH authorization mechanisms are registered by publishing a standards track or IESG-approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/urlauth-authorization-mechanism-registry This registry is case-insensitive. This document constitutes registration of the INTERNAL URLAUTH authorization mechanism. IMAP URLAUTH Authorization Mechanism Registry Mechanism Name Reference -------------- --------- INTERNAL [RFC4467] Crispin Standards Track [Page 16] RFC 4467 IMAP - URLAUTH Extension May 2006 12. Normative References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [BURL] Newman, C., "Message Submission BURL Extension", RFC 4468, May 2006. [DATETIME] Klyne, G. and C. Newman, "Date and Time on the Internet: Timestamps", RFC 3339, July 2002. [IMAP] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [IMAPURL] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. 13. Informative References [HMAC] Krawczyk, H., Bellare, M., and R. Canetti, "HMAC: Keyed- Hashing for Message Authentication", RFC 2104, February 1997. [RANDOM] Eastlake, D., 3rd, Schiller, J., and S. Crocker, "Randomness Requirements for Security", BCP 106, RFC 4086, June 2005. Author's Address Mark R. Crispin Networks and Distributed Computing University of Washington 4545 15th Avenue NE Seattle, WA 98105-4527 Phone: (206) 543-5762 EMail: MRC@CAC.Washington.EDU Crispin Standards Track [Page 17] RFC 4467 IMAP - URLAUTH Extension May 2006 Full Copyright Statement Copyright (C) The Internet Society (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is provided by the IETF Administrative Support Activity (IASA). Crispin Standards Track [Page 18] ================================================ FILE: docs/rfcs/rfc4469.IMAP_CATENATE_extension.txt ================================================ Network Working Group P. Resnick Request for Comments: 4469 QUALCOMM Incorporated Updates: 3501, 3502 April 2006 Category: Standards Track Internet Message Access Protocol (IMAP) CATENATE Extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2006). Abstract The CATENATE extension to the Internet Message Access Protocol (IMAP) extends the APPEND command to allow clients to create messages on the IMAP server that may contain a combination of new data along with parts of (or entire) messages already on the server. Using this extension, the client can catenate parts of an already existing message onto a new message without having to first download the data and then upload it back to the server. Resnick Standards Track [Page 1] RFC 4469 IMAP CATENATE Extension April 2006 1. Introduction The CATENATE extension to the Internet Message Access Protocol (IMAP) [1] allows the client to create a message on the server that can include the text of messages (or parts of messages) that already exist on the server without having to FETCH them and APPEND them back to the server. The CATENATE extension extends the APPEND command so that, instead of a single message literal, the command can take as arguments any combination of message literals (as described in IMAP [1]) and message URLs (as described in the IMAP URL Scheme [2] specification). The server takes all the pieces and catenates them into the output message. The CATENATE extension can also coexist with the MULTIAPPEND extension [3] to APPEND multiple messages in a single command. There are some obvious uses for the CATENATE extension. The motivating use case was to provide a way for a resource-constrained client to compose a message for subsequent submission that contains data that already exists in that client's IMAP store. Because the client does not have to download and re-upload potentially large message parts, bandwidth and processing limitations do not have as much impact. In addition, since the client can create a message in its own IMAP store, the command also addresses the desire of the client to archive a copy of a sent message without having to upload the message twice. (Mechanisms for sending the message are outside the scope of this document.) The extended APPEND command can also be used to copy parts of a message to another mailbox for archival purposes while getting rid of undesired parts. In environments where server storage is limited, a client could get rid of large message parts by copying over only the necessary parts and then deleting the original message. The mechanism could also be used to add data to a message (such as prepending message header fields) or to include other data by making a copy of the original and catenating the new data. 2. The CATENATE Capability A server that supports this extension returns "CATENATE" as one of the responses to the CAPABILITY command. Resnick Standards Track [Page 2] RFC 4469 IMAP CATENATE Extension April 2006 3. The APPEND Command Arguments: mailbox name (The following can be repeated in the presence of the MULTIAPPEND extension [3]) OPTIONAL flag parenthesized list OPTIONAL date/time string a single message literal or one or more message parts to catenate, specified as: message literal or message (or message part) URL Responses: OPTIONAL NO responses: BADURL, TOOBIG Result: OK - append completed NO - append error: can't append to that mailbox, error in flags or date/time or message text, or can't fetch that data BAD - command unknown or arguments invalid The APPEND command concatenates all the message parts and appends them as a new message to the end of the specified mailbox. The parenthesized flag list and date/time string set the flags and the internal date, respectively, as described in IMAP [1]. The subsequent command parameters specify the message parts that are appended sequentially to the output message. If the original form of APPEND is used, a message literal follows the optional flag list and date/time string, which is appended as described in IMAP [1]. If the extended form is used, "CATENATE" and a parenthesized list of message literals and message URLs follows, each of which is appended to the new message. If a message literal is specified (indicated by "TEXT"), the octets following the count are appended. If a message URL is specified (indicated by "URL"), the octets of the body part pointed to by that URL are appended, as if the literal returned in a FETCH BODY response were put in place of the message part specifier. The APPEND command does not cause the \Seen flag to be set for any catenated body part. The APPEND command does not change the selected mailbox. In the extended APPEND command, the string following "URL" is an IMAP URL [2] and is interpreted according to the rules of [2]. The present document only describes the behavior of the command using IMAP URLs that refer to specific messages or message parts on the current IMAP server from the current authenticated IMAP session. Because of that, only relative IMAP message or message part URLs (i.e., those having no scheme or ) are used. The base URL Resnick Standards Track [Page 3] RFC 4469 IMAP CATENATE Extension April 2006 for evaluating the relative URL is considered "imap://user@server/", where "user" is the user name of the currently authenticated user and "server" is the domain name of the current server. When in the selected state, the base URL is considered "imap://user@server/mailbox", where "mailbox" is the encoded name of the currently selected mailbox. Additionally, since the APPEND command is valid in the authenticated state of an IMAP session, no further LOGIN or AUTHENTICATE command is performed for URLs specified in the extended APPEND command. Note: Use of an absolute IMAP URL or any URL that refers to anything other than a message or message part from the current authenticated IMAP session is outside the scope of this document and would require an extension to this specification, and a server implementing only this specification would return NO to such a request. The client is responsible for making sure that the catenated message is in the format of an Internet Message Format (RFC 2822) [4] or Multipurpose Internet Mail Extension (MIME) [5] message. In particular, when a URL is catenated, the server copies octets, unchanged, from the indicated message or message part to the catenated message. It does no data conversion (e.g., MIME transfer encodings) nor any verification that the data is appropriate for the MIME part of the message into which it is inserted. The client is also responsible for inserting appropriate MIME boundaries between body parts, and writing MIME Content-Type and Content-Transfer- Encoding lines as needed in the appropriate places. Responses behave just as the original APPEND command described in IMAP [1]. If the server implements the IMAP UIDPLUS extension [6], it will also return an APPENDUID response code in the tagged OK response. Two response codes are provided in Section 4 that can be used in the tagged NO response if the APPEND command fails. 4. Response Codes When a APPEND command fails, it may return a response code that describes a reason for the failure. 4.1. BADURL Response The BADURL response code is returned if the APPEND fails to process one of the specified URLs. Possible reasons for this are bad URL syntax, unrecognized URL schema, invalid message UID, or invalid body part. The BADURL response code contains the first URL specified as a parameter to the APPEND command that has caused the operation to fail. Resnick Standards Track [Page 4] RFC 4469 IMAP CATENATE Extension April 2006 4.2. TOOBIG Response The TOOBIG response code is returned if the resulting message will exceed the 4-GB IMAP message limit. This might happen, for example, if the client specifies 3 URLs for 2-GB messages. Note that even if the server doesn't return TOOBIG, it still has to be defensive against misbehaving or malicious clients that try to construct a message over the 4-GB limit. The server may also wish to return the TOOBIG response code if the resulting message exceeds a server- specific message size limit. 5. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) [7] notation. Elements not defined here can be found in the formal syntax of the ABNF [7], IMAP [1], and IMAP ABNF extensions [8] specifications. Note that capability and resp-text-code are extended from the IMAP [1] specification and append-data is extended from the IMAP ABNF extensions [8] specification. append-data =/ "CATENATE" SP "(" cat-part *(SP cat-part) ")" cat-part = text-literal / url text-literal = "TEXT" SP literal url = "URL" SP astring resp-text-code =/ toobig-response-code / badurl-response-code toobig-response-code = "TOOBIG" badurl-response-code = "BADURL" SP url-resp-text url-resp-text = 1*(%x01-09 / %x0B-0C / %x0E-5B / %x5D-FE) ; Any TEXT-CHAR except "]" capability =/ "CATENATE" The astring in the definition of url and the url-resp-text in the definition of badurl-response-code each contain an imapurl as defined by [2]. Resnick Standards Track [Page 5] RFC 4469 IMAP CATENATE Extension April 2006 6. Acknowledgements Thanks to the members of the LEMONADE working group for their input. Special thanks to Alexey Melnikov for the examples. 7. Security Considerations The CATENATE extension does not raise any security considerations that are not present for the base protocol or in the use of IMAP URLs, and these issues are discussed in the IMAP [1] and IMAP URL [2] documents. 8. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG approved experimental RFC. The registry is currently located at . This document defines the CATENATE IMAP capability. The IANA has added this capability to the registry. Resnick Standards Track [Page 6] RFC 4469 IMAP CATENATE Extension April 2006 Appendix A. Examples Lines not starting with "C: " or "S: " are continuations of the previous lines. The original message in examples 1 and 2 below (UID = 20) has the following structure: multipart/mixed MIME message with two body parts: 1. text/plain 2. application/x-zip-compressed Example 1: The following example demonstrates how a CATENATE client can replace an attachment in a draft message, without the need to download it to the client and upload it back. C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE (URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=HEADER" TEXT {42} S: + Ready for literal data C: C: --------------030308070208000400050907 C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1.MIME" URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;section=1" TEXT {42} S: + Ready for literal data C: C: --------------030308070208000400050907 C: URL "/Drafts;UIDVALIDITY=385759045/;UID=30" TEXT {44} S: + Ready for literal data C: C: --------------030308070208000400050907-- C: ) S: A003 OK catenate append completed Resnick Standards Track [Page 7] RFC 4469 IMAP CATENATE Extension April 2006 Example 2: The following example demonstrates how the CATENATE extension can be used to replace edited text in a draft message, as well as header fields for the top level message part (e.g., Subject has changed). The previous version of the draft is marked as \Deleted. Note that the server also supports the UIDPLUS extension, so the APPENDUID response code is returned in the successful OK response to the APPEND command. C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE (TEXT {738} S: + Ready for literal data C: Return-Path: C: Received: from [127.0.0.2] C: by rufus.example.org via TCP (internal) with ESMTPA; C: Thu, 11 Nov 2004 16:57:07 +0000 C: Message-ID: <419399E1.6000505@example.org> C: Date: Thu, 12 Nov 2004 16:57:05 +0000 C: From: Bob Ar C: X-Accept-Language: en-us, en C: MIME-Version: 1.0 C: To: foo@example.net C: Subject: About our holiday trip C: Content-Type: multipart/mixed; C: boundary="------------030308070208000400050907" C: C: --------------030308070208000400050907 C: Content-Type: text/plain; charset=us-ascii; format=flowed C: Content-Transfer-Encoding: 7bit C: C: Our travel agent has sent the updated schedule. C: C: Cheers, C: Bob C: --------------030308070208000400050907 C: URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2.MIME" URL "/Drafts;UIDVALIDITY=385759045/;UID=20/;Section=2" TEXT {44} S: + Ready for literal data C: C: --------------030308070208000400050907-- C: ) S: A003 OK [APPENDUID 385759045 45] append Completed C: A004 UID STORE 20 +FLAGS.SILENT (\Deleted) S: A004 OK STORE completed Resnick Standards Track [Page 8] RFC 4469 IMAP CATENATE Extension April 2006 Example 3: The following example demonstrates how the CATENATE extension can be used to strip attachments. Below, a PowerPoint attachment was replaced by a small text part explaining that the attachment was stripped. C: A003 APPEND Drafts (\Seen \Draft $MDNSent) CATENATE (URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=HEADER" TEXT {42} S: + Ready for literal data C: C: --------------030308070208000400050903 C: URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1.MIME" URL "/Drafts;UIDVALIDITY=385759045/;UID=21/;section=1" TEXT {255} S: + Ready for literal data C: C: --------------030308070208000400050903 C: Content-type: text/plain; charset="us-ascii" C: Content-transfer-encoding: 7bit C: C: This body part contained a Power Point presentation that was C: deleted upon your request. C: --------------030308070208000400050903-- C: ) S: A003 OK append Completed Resnick Standards Track [Page 9] RFC 4469 IMAP CATENATE Extension April 2006 Example 4: The following example demonstrates a failed APPEND command. The server returns the BADURL response code to indicate that one of the provided URLs is invalid. This example also demonstrates how the CATENATE extension can be used to construct a digest of several messages. C: A003 APPEND Sent (\Seen $MDNSent) CATENATE (TEXT {541} S: + Ready for literal data C: Return-Path: C: Received: from [127.0.0.2] C: by rufus.example.org via TCP (internal) with ESMTPA; C: Thu, 11 Nov 2004 16:57:07 +0000 C: Message-ID: <419399E1.6000505@example.org> C: Date: Thu, 21 Nov 2004 16:57:05 +0000 C: From: Farren Oo C: X-Accept-Language: en-us, en C: MIME-Version: 1.0 C: To: bar@example.org C: Subject: Digest of the mailing list for today C: Content-Type: multipart/digest; C: boundary="------------030308070208000400050904" C: C: --------------030308070208000400050904 C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11467" TEXT {42} S: + Ready for literal data C: C: --------------030308070208000400050904 C: URL "/INBOX;UIDVALIDITY=785799047/;UID=113330/;section=1.5.9" TEXT {42} S: + Ready for literal data C: C: --------------030308070208000400050904 C: URL "/INBOX;UIDVALIDITY=785799047/;UID=11916" TEXT {44} S: + Ready for literal data C: C: --------------030308070208000400050904-- C: ) S: A003 NO [BADURL "/INBOX;UIDVALIDITY=785799047/;UID=113330; section=1.5.9"] CATENATE append has failed, one message expunged Note that the server could have validated the URLs as they were received and therefore could have returned the tagged NO response with BADURL response-code in place of any continuation request after the URL was received. Resnick Standards Track [Page 10] RFC 4469 IMAP CATENATE Extension April 2006 9. Normative References [1] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [2] Newman, C., "IMAP URL Scheme", RFC 2192, September 1997. [3] Crispin, M., "Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension", RFC 3502, March 2003. [4] Resnick, P., "Internet Message Format", RFC 2822, April 2001. [5] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [6] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS extension", RFC 4315, December 2005. [7] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [8] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. Resnick Standards Track [Page 11] RFC 4469 IMAP CATENATE Extension April 2006 Author's Address Peter W. Resnick QUALCOMM Incorporated 5775 Morehouse Drive San Diego, CA 92121-1714 US Phone: +1 858 651 4478 EMail: presnick@qualcomm.com URI: http://www.qualcomm.com/~presnick/ Resnick Standards Track [Page 12] RFC 4469 IMAP CATENATE Extension April 2006 Full Copyright Statement Copyright (C) The Internet Society (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is provided by the IETF Administrative Support Activity (IASA). Resnick Standards Track [Page 13] ================================================ FILE: docs/rfcs/rfc4549.Sync_operations_for_disconnected_IMAP4_Clients.txt ================================================ Network Working Group A. Melnikov, Ed. Request for Comments: 4549 Isode Ltd. Category: Informational June 2006 Synchronization Operations for Disconnected IMAP4 Clients Status of This Memo This memo provides information for the Internet community. It does not specify an Internet standard of any kind. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2006). Abstract This document attempts to address some of the issues involved in building a disconnected IMAP4 client. In particular, it deals with the issues of what might be called the "driver" portion of the synchronization tool: the portion of the code responsible for issuing the correct set of IMAP4 commands to synchronize the disconnected client in the way that is most likely to make the human who uses the disconnected client happy. This note describes different strategies that can be used by disconnected clients and shows how to use IMAP protocol in order to minimize the time of the synchronization process. This note also lists IMAP extensions that a server should implement in order to provide better synchronization facilities to disconnected clients. Melnikov Informational [Page 1] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Table of Contents 1. Introduction ....................................................3 1.1. Conventions Used in This Document ..........................3 2. Design Principles ...............................................3 3. Overall Picture of Synchronization ..............................4 4. Mailbox Synchronization Steps and Strategies ....................7 4.1. Checking UID Validity ......................................7 4.2. Synchronizing Local Changes with the Server ................8 4.2.1. Uploading Messages to the Mailbox ...................8 4.2.2. Optimizing "move" and "copy" Operations .............9 4.2.3. Replaying Local Flag Changes .......................14 4.2.4. Processing Mailbox Compression (EXPUNGE) Requests ..15 4.2.5. Closing a Mailbox ..................................17 4.3. Details of "Normal" Synchronization of a Single Mailbox ...18 4.3.1. Discovering New Messages and Changes to Old Messages ...........................................18 4.3.2. Searching for "Interesting" Messages. ..............20 4.3.3. Populating Cache with "Interesting" Messages. ......21 4.3.4. User-Initiated Synchronization .....................22 4.4. Special Case: Descriptor-Only Synchronization .............22 4.5. Special Case: Fast New-Only Synchronization ...............23 4.6. Special Case: Blind FETCH .................................23 5. Implementation Considerations ..................................24 5.1. Error Recovery during Playback ............................26 5.2. Quality of Implementation Issues ..........................28 5.3. Optimizations .............................................28 6. IMAP Extensions That May Help ..................................30 6.1. CONDSTORE Extension .......................................30 7. Security Considerations ........................................33 8. References .....................................................33 8.1. Normative References ......................................33 8.2. Informative References ....................................34 9. Acknowledgements ...............................................34 Melnikov Informational [Page 2] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 1. Introduction Several recommendations presented in this document are generally applicable to all types of IMAP clients. However, this document tries to concentrate on disconnected mail clients [IMAP-MODEL]. It also suggests some IMAP extensions* that should be implemented by IMAP servers in order to make the life of disconnected clients easier. In particular, the [UIDPLUS] extension was specifically designed to streamline certain disconnected operations, like expunging, uploading, and copying messages (see Sections 4.2.1, 4.2.2.1, and 4.2.4). Readers of this document are also strongly advised to read RFC 2683 [RFC2683]. * Note that the functionality provided by the base IMAP protocol [IMAP4] is sufficient to perform basic synchronization. 1.1. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. Long lines in examples are broken for editorial clarity. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [KEYWORDS]. Let's call an IMAP command idempotent if the result of executing the command twice sequentially is the same as the result of executing the command just once. 2. Design Principles All mailbox state or content information stored on the disconnected client should be viewed strictly as a cache of the state of the server. The "master" state remains on the server, just as it would with an interactive IMAP4 client. The one exception to this rule is that information about the state of the disconnected client's cache (the state includes flag changes while offline and during scheduled message uploads) remains on the disconnected client: that is, the IMAP4 server is not responsible for remembering the state of the disconnected IMAP4 client. We assume that a disconnected client is a client that, for whatever reason, wants to minimize the length of time that it is "on the phone" to the IMAP4 server. Often this will be because the client is using a dialup connection, possibly with very low bandwidth, but Melnikov Informational [Page 3] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 sometimes it might just be that the human is in a hurry to catch an airplane, or some other event beyond our control. Whatever the reason, we assume that we must make efficient use of the network connection, both in the usual sense (not generating spurious traffic) and in the sense that we would prefer not to have the connection sitting idle while the client and/or the server is performing strictly local computation or I/O. Another, perhaps simpler way of stating this is that we assume that network connections are "expensive". Practical experience with disconnected mail systems has shown that there is no single synchronization strategy that is appropriate for all cases. Different humans have different preferences, and the same human's preference will vary depending both on external circumstance (how much of a hurry the human is in today) and on the value that the human places on the messages being transferred. The point here is that there is no way that the synchronization program can guess exactly what the human wants to do, so the human will have to provide some guidance. Taken together, the preceding two principles lead to the conclusion that the synchronization program must make its decisions based on some kind of guidance provided by the human, by selecting the appropriate options in the user interface or through some sort of configuration file. Almost certainly, it should not pause for I/O with the human in the middle of the synchronization process. The human will almost certainly have several different configurations for the synchronization program, for different circumstances. Since a disconnected client has no way of knowing what changes might have occurred to the mailbox while it was disconnected, message numbers are not useful to a disconnected client. All disconnected client operations should be performed using UIDs, so that the client can be sure that it and the server are talking about the same messages during the synchronization process. 3. Overall Picture of Synchronization The basic strategy for synchronization is outlined below. Note that the real strategy may vary from one application to another or may depend on a synchronization mode. a) Process any "actions" that were pending on the client that were not associated with any mailbox. (In particular sending messages composed offline with SMTP. This is not part of IMAP synchronization, but it is mentioned here for completeness.) Melnikov Informational [Page 4] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 b) Fetch the current list of "interesting" mailboxes. (The disconnected client should allow the user to skip this step completely.) c) "Client-to-server synchronization": for each IMAP "action" that was pending on the client, do the following: 1) If the action implies opening a new mailbox (any operation that operates on messages), open the mailbox. Check its UID validity value (see Section 4.1 for more details) returned in the UIDVALIDITY response code. If the UIDVALIDITY value returned by the server differs, the client MUST empty the local cache of the mailbox and remove any pending "actions" that refer to UIDs in that mailbox (and consider them failed). Note that this doesn't affect actions performed on client-generated fake UIDs (see Section 5). 2) Perform the action. If the action is to delete a mailbox (DELETE), make sure that the mailbox is closed first (see also Section 3.4.12 of [RFC2683]). d) "Server-to-client synchronization": for each mailbox that requires synchronization, do the following: 1) Check the mailbox UIDVALIDITY (see Section 4.1 for more details) with SELECT/EXAMINE/STATUS. If UIDVALIDITY value returned by the server differs, the client MUST * empty the local cache of that mailbox; * remove any pending "actions" that refer to UIDs in that mailbox and consider them failed; and * skip step 2-II. 2) Fetch the current "descriptors"; I) Discover new messages. II) Discover changes to old messages. 3) Fetch the bodies of any "interesting" messages that the client doesn't already have. e) Close all open mailboxes not required for further operations (if staying online) or disconnect all open connections (if going offline). Melnikov Informational [Page 5] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Terms used: "Actions" are queued requests that were made by the human to the client's Mail User Agent (MUA) software while the client was disconnected. We define "descriptors" as a set of IMAP4 FETCH data items. Conceptually, a message's descriptor is that set of information that allows the synchronization program to decide what protocol actions are necessary to bring the local cache to the desired state for this message; since this decision is really up to the human, this information probably includes at least a few header fields intended for human consumption. Exactly what will constitute a descriptor depends on the client implementation. At a minimum, the descriptor contains the message's UID and FLAGS. Other likely candidates are the RFC822.SIZE, RFC822.HEADER, BODYSTRUCTURE, or ENVELOPE data items. Comments: 1) The list of actions should be ordered. For example, if the human deletes message A1 in mailbox A, then expunges mailbox A, and then deletes message A2 in mailbox A, the human will expect that message A1 is gone and that message A2 is still present but is now deleted. By processing all the actions before proceeding with synchronization, we avoid having to compensate for the local MUA's changes to the server's state. That is, once we have processed all the pending actions, the steps that the client must take to synchronize itself will be the same no matter where the changes to the server's state originated. 2) Steps a and b can be performed in parallel. Alternatively, step a can be performed after d. 3) On step b, the set of "interesting" mailboxes pretty much has to be determined by the human. What mailboxes belong to this set may vary between different IMAP4 sessions with the same server, client, and human. An interesting mailbox can be a mailbox returned by LSUB command (see Section 6.3.9 of [IMAP4]). The special mailbox "INBOX" SHOULD be in the default set of mailboxes that the client considers interesting. However, providing the ability to ignore INBOX for a particular session or client may be valuable for some mail filtering strategies. Melnikov Informational [Page 6] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 4) On step d-2-II, the client also finds out about changes to the flags of messages that the client already has in its local cache, and about messages in the local cache that no longer exist on the server (i.e., messages that have been expunged). 5) "Interesting" messages are those messages that the synchronization program thinks the human wants to have cached locally, based on the configuration and the data retrieved in step b. 6) A disconnected IMAP client is a special case of an IMAP client, so it MUST be able to handle any "unexpected" unsolicited responses, like EXISTS and EXPUNGE, at any time. The disconnected client MAY ignore EXPUNGE response during "client-to-server" synchronization phase (step c). The rest of this discussion will focus primarily on the synchronization issues for a single mailbox. 4. Mailbox Synchronization Steps and Strategies 4.1. Checking UID Validity The "UID validity" of a mailbox is a number returned in an UIDVALIDITY response code in an OK untagged response at mailbox selection time. The UID validity value changes between sessions when UIDs fail to persist between sessions. Whenever the client selects a mailbox, the client must compare the returned UID validity value with the value stored in the local cache. If the UID validity values differ, the UIDs in the client's cache are no longer valid. The client MUST then empty the local cache of that mailbox and remove any pending "actions" that refer to UIDs in that mailbox. The client MAY also issue a warning to the human. The client MUST NOT cancel any scheduled uploads (i.e., APPENDs) for the mailbox. Note that UIDVALIDITY is not only returned on a mailbox selection. The COPYUID and APPENDUID response codes defined in the [UIDPLUS] extension (see also 4.2.2) and the UIDVALIDITY STATUS response data item also contain a UIDVALIDITY value for some other mailbox. The client SHOULD behave as described in the previous paragraph (but it should act on the other mailbox's cache), no matter how it obtained the UIDVALIDITY value. Melnikov Informational [Page 7] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 4.2. Synchronizing Local Changes with the Server 4.2.1. Uploading Messages to the Mailbox Two of the most common examples of operations resulting in message uploads are: 1) Saving a draft message 2) Copying a message between remote mailboxes on two different IMAP servers or a local mailbox and a remote mailbox. Message upload is performed with the APPEND command. A message scheduled to be uploaded has no UID associated with it, as all UIDs are assigned by the server. The APPEND command will effectively associate a UID with the uploaded message that can be stored in the local cache for future reference. However, [IMAP4] doesn't describe a simple mechanism to discover the message UID by just performing the APPEND command. In order to discover the UID, the client can do one of the following: 1) Remove the uploaded message from cache. Then, use the mechanism described in 4.3 to fetch the information about the uploaded message as if it had been uploaded by some other client. 2) Try to fetch header information as described in 4.2.2 in order to find a message that corresponds to the uploaded message. One strategy for doing this is described in 4.2.2. Case 1 describes a not particularly smart client. C: A003 APPEND Drafts (\Seen $MDNSent) {310} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK APPEND Completed Fortunately, there is a simpler way to discover the message UID in the presence of the [UIDPLUS] extension: Melnikov Informational [Page 8] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 C: A003 APPEND Drafts (\Seen $MDNSent) {310} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK [APPENDUID 1022843275 77712] APPEND completed The UID of the appended message is the second parameter of APPENDUID response code. 4.2.2. Optimizing "move" and "copy" Operations Practical experience with IMAP and other mailbox access protocols that support multiple mailboxes suggests that moving a message from one mailbox to another is an extremely common operation. 4.2.2.1. Moving a Message between Two Mailboxes on the Same Server In IMAP4, a "move" operation between two mailboxes on the same server is really a combination of a COPY operation and a STORE +FLAGS (\Deleted) operation. This makes good protocol sense for IMAP, but it leaves a simple-minded disconnected client in the silly position of deleting and possibly expunging its cached copy of a message, then fetching an identical copy via the network. However, the presence of the UIDPLUS extension in the server can help: C: A001 UID COPY 567,414 "Interesting Messages" S: A001 OK [COPYUID 1022843275 414,567 5:6] Completed This tells the client that the message with UID 414 in the current mailbox was successfully copied to the mailbox "Interesting Messages" and was given the UID 5, and that the message with UID 567 was given the UID 6. In the absence of UIDPLUS extension support in the server, the following trick can be used. By including the Message-ID: header and the INTERNALDATE data item as part of the descriptor, the client can check the descriptor of a "new" message against messages that are already in its cache and avoid fetching the extra copy. Of course, Melnikov Informational [Page 9] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 it's possible that the cost of checking to see if the message is already in the local cache may exceed the cost of just fetching it, so this technique should not be used blindly. If the MUA implements a "move" command, it makes special provisions to use this technique when it knows that a copy/delete sequence is the result of a "move" command. Note that servers are not required (although they are strongly encouraged with "SHOULD language") to preserve INTERNALDATE when copying messages. Also note that since it's theoretically possible for this algorithm to find the wrong message (given sufficiently malignant Message-ID headers), implementers should provide a way to disable this optimization, both permanently and on a message-by-message basis. Example 1: Copying a message in the absence of UIDPLUS extension. At some point in time the client has fetched the source message and some information was cached: C: C021 UID FETCH (BODY.PEEK[] INTERNALDATE FLAGS) ... S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" FLAGS (\Draft $MDNSent) BODY[] {1036} S: ... S: Message-Id: <20040903110856.22a127cd@chardonnay> S: ... S: ...message body... S: ) ... S: C021 OK fetch completed Later on, the client decides to copy the message: C: C035 UID COPY 123 "Interesting Messages" S: C035 OK Completed As the server hasn't provided the COPYUID response code, the client tries the optimization described above: C: C036 SELECT "Interesting Messages" ... C: C037 UID SEARCH ON 31-May-2002 HEADER "Message-Id" "20040903110856.22a127cd@chardonnay" S: SEARCH 12368 S: C037 OK completed Melnikov Informational [Page 10] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Note that if the server has returned multiple UIDs in the SEARCH response, the client MUST NOT use any of the returned UID. 4.2.2.2. Moving a Message from a Remote Mailbox to a Local Moving a message from a remote mailbox to a local is done with FETCH (that includes FLAGS and INTERNALDATE) followed by UID STORE +FLAGS.SILENT (\Deleted): C: A003 UID FETCH 123 (BODY.PEEK[] INTERNALDATE FLAGS) S: * 27 FETCH (UID 123 INTERNALDATE "31-May-2002 05:26:59 -0600" FLAGS (\Seen $MDNSent) BODY[] S: ...message body... S: ) S: A003 OK UID FETCH completed C: A004 UID STORE +FLAGS.SILENT (\Deleted) S: A004 STORE completed Note that there is no reason to fetch the message during synchronization if it's already in the client's cache. Also, the client SHOULD preserve delivery date in the local cache. 4.2.2.3. Moving a Message from a Local Mailbox to a Remote Moving a message from a local mailbox to a remote is done with APPEND: C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" {310} S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK [APPENDUID 1022843275 77712] completed The client SHOULD specify the delivery date from the local cache in the APPEND. If the [LITERAL+] extension is available, the client can save a round-trip*: Melnikov Informational [Page 11] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" {310+} C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: S: A003 OK [APPENDUID 1022843275 77712] completed * Note that there is a risk that the server will reject the message due to its size. If this happens, the client will waste bandwidth transferring the whole message. If the client wouldn't have used the LITERAL+, this could have been avoided: C: A003 APPEND Drafts (\Seen $MDNSent) "31-May-2004 05:26:59 -0600" {16777215} S: A003 NO Sorry, message is too big 4.2.2.4. Moving a Message between Two Mailboxes on Different Servers Moving a message between two mailbox on two different servers is a combination of the operations described in 4.2.2.2 followed by the operations described in 4.2.2.3. 4.2.2.5. Uploading Multiple Messages to a Remote Mailbox with MULTIAPPEND When there is a need to upload multiple messages to a remote mailbox (e.g., as per 4.2.2.3), the presence of certain IMAP extensions may significantly improve performance. One of them is [MULTIAPPEND]. For some mail stores, opening a mailbox for appending might be expensive. [MULTIAPPEND] tells the server to open the mailbox once (instead of opening and closing it "n" times per "n" messages to be uploaded) and to keep it open while a group of messages is being uploaded to the server. Also, if the server supports both [MULTIAPPEND] and [LITERAL+] extensions, the entire upload is accomplished in a single command/response round-trip. Melnikov Informational [Page 12] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Note: Client implementers should be aware that [MULTIAPPEND] performs append of multiple messages atomically. This means, for example, if there is not enough space to save "n"-th message (or the message has invalid structure and is rejected by the server) after successful upload of "n-1" messages, the whole upload operation fails, and no message will be saved in the mailbox. Although this behavior might be desirable in certain situations, it might not be what you want. Otherwise, the client should use the regular APPEND command (Section 4.2.2.3), possibly utilizing the [LITERAL+] extension. See also Section 5.1 for discussions about error recovery. Note: MULTIAPPEND can be used together with the UIDPLUS extension in a way similar to what was described in Section 4.2.1. [MULTIAPPEND] extends the syntax of the APPENDUID response code to allow for multiple message UIDs in the second parameter. Example 2: This example demonstrates the use of MULTIAPPEND together with UIDPLUS (synchronization points where the client waits for confirmations from the server are marked with "<--->"): C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" {310} <---> S: + Ready for literal data C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286} <---> S: + Ready for literal data C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) C: From: Joe Mooch C: Subject: Re: afternoon meeting C: To: foobar@blt.example.com C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: 3:30 is fine with me. C: Melnikov Informational [Page 13] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 S: A003 OK [APPENDUID 1022843275 77712,77713] completed The upload takes 3 round-trips. Example 3: In this example, Example 2 was modified for the case when the server supports MULTIAPPEND, LITERAL+, and UIDPLUS. The upload takes only 1 round-trip. C: A003 APPEND Jan-2002 (\Seen $MDNSent) "31-May-2002 05:26:59 -0600" {310+} C: Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST) C: From: Fred Foobar C: Subject: afternoon meeting C: To: mooch@owatagu.siam.edu C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: Hello Joe, do you think we can meet at 3:30 tomorrow? C: (\Seen) " 1-Jun-2002 22:43:04 -0800" {286+} C: Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST) C: From: Joe Mooch C: Subject: Re: afternoon meeting C: To: foobar@blt.example.com C: Message-Id: C: MIME-Version: 1.0 C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII C: C: 3:30 is fine with me. C: S: A003 OK [APPENDUID 1022843275 77712,77713] completed 4.2.3. Replaying Local Flag Changes The disconnected client uses the STORE command to synchronize local flag state with the server. The disconnected client SHOULD use +FLAGS.SILENT or -FLAGS.SILENT in order to set or unset flags modified by the user while offline. The FLAGS form MUST NOT be used, as there is a risk that this will overwrite flags on the server that have been changed by some other client. Example 4: For the message with UID 15, the disconnected client stores the following flags \Seen and $Highest. The flags were modified on the server by some other client: \Seen, \Answered, and $Highest. While Melnikov Informational [Page 14] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 offline, the user requested that the $Highest flags be removed and that the \Deleted flag be added. The flag synchronization sequence for the message should look like: C: A001 UID STORE 15 +FLAGS.SILENT (\Deleted) S: A001 STORE completed C: A002 UID STORE 15 -FLAGS.SILENT ($Highest) S: A002 STORE completed If the disconnected client is able to store an additional binary state information (or a piece of information that can take a value from a predefined set of values) in the local cache of an IMAP mailbox or in a local mailbox (e.g., message priority), and if the server supports storing of arbitrary keywords, the client MUST use keywords to store this state on the server. Example 5: Imagine a speculative mail client that can mark a message as one of work-related ($Work), personal ($Personal), or spam ($Spam). In order to mark a message as personal, the client issues: C: A001 UID STORE 15 +FLAGS.SILENT ($Personal) S: A001 STORE completed C: A002 UID STORE 15 -FLAGS.SILENT ($Work $Spam) S: A002 STORE completed In order to mark the message as not work, not personal and not spam, the client issues: C: A003 UID STORE 15 -FLAGS.SILENT ($Personal $Work $Spam) S: A003 STORE completed 4.2.4. Processing Mailbox Compression (EXPUNGE) Requests A naive disconnected client implementation that supports compressing a mailbox while offline may decide to issue an EXPUNGE command to the server in order to expunge messages marked \Deleted. The problem with this command during synchronization is that it permanently erases all messages with the \Deleted flag set, i.e., even those messages that were marked as \Deleted on the server while the user was offline. Doing this might result in an unpleasant surprise for the user. Fortunately the [UIDPLUS] extension can help in this case as well. The extension introduces UID EXPUNGE command, that, unlike EXPUNGE, takes a UID set parameter, that lists UIDs of all messages that can be expunged. When processing this command the server erases only Melnikov Informational [Page 15] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 messages with \Deleted flag listed in the UID list. Thus, messages not listed in the UID set will not be expunged even if they have the \Deleted flag set. Example 6: While the user was offline, 3 messages with UIDs 7, 27, and 65 were marked \Deleted when the user requested to compress the open mailbox. Another client marked a message \Deleted on the server (UID 34). During synchronization, the disconnected client issues: C: A001 UID EXPUNGE 7,27,65 S: * ... EXPUNGE S: * ... EXPUNGE S: * ... EXPUNGE S: A001 UID EXPUNGE completed If another client issues UID SEARCH DELETED command (to find all messages with the \Deleted flag) before and after the UID EXPUNGE, it will get: Before: C: B001 UID SEARCH DELETED S: * SEARCH 65 34 27 7 S: B001 UID SEARCH completed After: C: B002 UID SEARCH DELETED S: * SEARCH 34 S: B002 UID SEARCH completed In the absence of the [UIDPLUS] extension, the following sequence of commands can be used as an approximation. Note: It's possible for another client to mark additional messages as deleted while this sequence is being performed. In this case, these additional messages will be expunged as well. 1) Find all messages marked \Deleted on the server. C: A001 UID SEARCH DELETED S: * SEARCH 65 34 27 7 S: A001 UID SEARCH completed 2) Find all messages that must not be erased (for the previous example the list will consist of the message with UID 34). Melnikov Informational [Page 16] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 3) Temporarily remove \Deleted flag on all messages found in step 2. C: A002 UID STORE 34 -FLAGS.SILENT (\Deleted) S: A002 UID STORE completed 4) Expunge the mailbox. C: A003 EXPUNGE S: * 20 EXPUNGE S: * 7 EXPUNGE S: * 1 EXPUNGE S: A003 EXPUNGE completed Here, the message with UID 7 has message number 1, with UID 27 has message number 7, and with UID 65 has message number 20. 5) Restore \Deleted flag on all messages found when performing step 2. C: A004 UID STORE 34 +FLAGS.SILENT (\Deleted) S: A004 UID STORE completed 4.2.5. Closing a Mailbox When the disconnected client has to close a mailbox, it should not use the CLOSE command, because CLOSE does a silent EXPUNGE. (Section 4.2.4 explains why EXPUNGE should not be used by a disconnected client.) It is safe to use CLOSE only if the mailbox was opened with EXAMINE. If the mailbox was opened with SELECT, the client can use one of the following commands to implicitly close the mailbox and prevent the silent expunge: 1) UNSELECT - This is a command described in [UNSELECT] that works as CLOSE, but doesn't cause the silent EXPUNGE. This command is supported by the server if it reports UNSELECT in its CAPABILITY list. 2) SELECT - SELECT causes implicit CLOSE without EXPUNGE. 3) If the client intends to issue LOGOUT after closing the mailbox, it may just issue LOGOUT, because LOGOUT causes implicit CLOSE without EXPUNGE as well. 4) SELECT - If the client knows a mailbox that doesn't exist or can't be selected, it MAY SELECT it. Melnikov Informational [Page 17] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 If the client opened the mailbox with SELECT and just wants to avoid implicit EXPUNGE without closing the mailbox, it may also use the following: 5) EXAMINE - Reselect the same mailbox in read-only mode. 4.3. Details of "Normal" Synchronization of a Single Mailbox The most common form of synchronization is where the human trusts the integrity of the client's copy of the state of a particular mailbox and simply wants to bring the client's cache up to date so that it accurately reflects the mailbox's current state on the server. 4.3.1. Discovering New Messages and Changes to Old Messages Let represent the highest UID that the client knows about in this mailbox. Since UIDs are allocated in strictly ascending order, this is simply the UID of the last message in the mailbox that the client knows about. Let represent 's UID plus one. Let represent a list consisting of all the FETCH data item items that the implementation considers part of the descriptor; at a minimum this is just the FLAGS data item, but it usually also includes BODYSTRUCTURE and RFC822.SIZE. At this step, SHOULD NOT include RFC822. With no further information, the client can issue the following two commands: tag1 UID FETCH :* tag2 UID FETCH 1: FLAGS The first command will request some information about "new" messages (i.e., messages received by the server since the last synchronization). It will also allow the client to build a message number to UID map (only for new messages). The second command allows the client to 1) update cached flags for old messages; 2) find out which old messages got expunged; and 3) build a mapping between message numbers and UIDs (for old messages). The order here is significant. We want the server to start returning the list of new message descriptors as fast as it can, so that the client can start issuing more FETCH commands, so we start out by asking for the descriptors of all the messages we know the client Melnikov Informational [Page 18] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 cannot possibly have cached yet. The second command fetches the information we need to determine what changes may have occurred to messages that the client already has cached. Note that the former command should only be issued if the UIDNEXT value cached by the client differs from the one returned by the server. Once the client has issued these two commands, there's nothing more the client can do with this mailbox until the responses to the first command start arriving. A clever synchronization program might use this time to fetch its local cache state from disk or to start the process of synchronizing another mailbox. The following is an example of the first FETCH: C: A011 UID fetch 131:* (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) Note 1: The first FETCH may result in the server's sending a huge volume of data. A smart disconnected client should use message ranges (see also Section 3.2.1.2 of [RFC2683]), so that the user is able to execute a different operation between fetching information for a group of new messages. Example 7: Knowing the new UIDNEXT returned by the server on SELECT or EXAMINE (), the client can split the UID range : into groups, e.g., 100 messages. After that, the client can issue: C: A011 UID fetch : (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) ... C: A012 UID fetch : (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) ... ... C: A0FF UID fetch : (FLAGS BODYSTRUCTURE INTERNALDATE RFC822.SIZE) Note that unless a SEARCH command is issued, it is impossible to determine how many messages will fall into a subrange, as UIDs are not necessarily contiguous. Note 2: The client SHOULD ignore any unsolicited EXPUNGE responses received during the first FETCH command. EXPUNGE responses contain message numbers that are useless to a client that doesn't have the message-number-to-UID translation table. Melnikov Informational [Page 19] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 The second FETCH command will result in zero or more untagged fetch responses. Each response will have a corresponding UID FETCH data item. All messages that didn't have a matching untagged FETCH response MUST be removed from the local cache. For example, if the had a value 15000 and the local cache contained 3 messages with the UIDs 12, 777, and 14999, respectively, then after receiving the following responses from the server, the client must remove the message with UID 14999 from its local cache. S: * 1 FETCH (UID 12 FLAGS (\Seen)) S: * 2 FETCH (UID 777 FLAGS (\Answered \Deleted)) Note 3: If the client is not interested in flag changes (i.e., the client only wants to know which old messages are still on the server), the second FETCH command can be substituted with: tag2 UID SEARCH UID 1: This command will generate less traffic. However, an implementor should be aware that in order to build the mapping table from message numbers to UIDs, the output of the SEARCH command MUST be sorted first, because there is no requirement for a server to return UIDs in SEARCH response in any particular order. 4.3.2. Searching for "Interesting" Messages. This step is performed entirely on the client (from the information received in the step described in 4.3.1), entirely on the server, or on some combination of both. The decision on what is an "interesting" message is up to the client software and the human. One easy criterion that should probably be implemented in any client is whether the message is "too big" for automatic retrieval, where "too big" is a parameter defined in the client's configuration. Another commonly used criterion is the age of a message. For example, the client may choose to download only messages received in the last week (in this case, would be today's date minus 7 days): tag3 UID SEARCH UID SINCE Keep in mind that a date search disregards time and time zone. The client can avoid doing this search if it specified INTERNALDATE in on the step described in 4.3.1. If the client did, it can perform the local search on its message cache. Melnikov Informational [Page 20] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 At this step, the client also decides what kind of information about a particular message to fetch from the server. In particular, even for a message that is considered "too big", the client MAY choose to fetch some part(s) of it. For example, if the message is a multipart/mixed containing a text part and a MPEG attachment, there is no reason for the client not to fetch the text part. The decision of which part should or should not be fetched can be based on the information received in the BODYSTRUCTURE FETCH response data item (i.e., if BODYSTRUCTURE was included in on the step described in 4.3.1). 4.3.3. Populating Cache with "Interesting" Messages. Once the client has found out which messages are "interesting", it can start issuing appropriate FETCH commands for "interesting" messages or parts thereof. Note that fetching a message into the disconnected client's local cache does NOT imply that the human has (or even will) read the message. Thus, the synchronization program for a disconnected client should always be careful to use the .PEEK variants of the FETCH data items that implicitly set the \Seen flag. Once the last descriptor has arrived and the last FETCH command has been issued, the client simply needs to process the incoming fetch items and use them to update the local message cache. In order to avoid deadlock problems, the client must give processing of received messages priority over issuing new FETCH commands during this synchronization process. This may necessitate temporary local queuing of FETCH requests that cannot be issued without causing a deadlock. In order to achieve the best use of the "expensive" network connection, the client will almost certainly need to pay careful attention to any flow-control information that it can obtain from the underlying transport connection (usually a TCP connection). Note: The requirement stated in the previous paragraph might result in an unpleasant user experience, if followed blindly. For example, the user might be unwilling to wait for the client to finish synchronization before starting to process the user's requests. A smart disconnected client should allow the user to perform requested operations in between IMAP commands that are part of the synchronization process. See also Note 1 in Section 4.3.1. Melnikov Informational [Page 21] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Example 8: After fetching a message BODYSTRUCTURE, the client discovers a complex MIME message. Then, it decides to fetch MIME headers of the nested MIME messages and some body parts. C: A011 UID fetch 11 (BODYSTRUCTURE) S: ... C: A012 UID fetch 11 (BODY[HEADER] BODY[1.MIME] BODY[1.1.MIME] BODY[1.2.MIME] BODY[2.MIME] BODY[3.MIME] BODY[4.MIME] BODY[5.MIME] BODY[6.MIME] BODY[7.MIME] BODY[8.MIME] BODY[9.MIME] BODY[10.MIME] BODY[11.MIME] BODY[12.MIME] BODY[13.MIME] BODY[14.MIME] BODY[15.MIME] BODY[16.MIME] BODY[17.MIME] BODY[18.MIME] BODY[19.MIME] BODY[20.MIME] BODY[21.MIME]) S: ... C: A013 UID fetch 11 (BODY[1.1] BODY[1.2]) S: ... C: A014 UID fetch 11 (BODY[3] BODY[4] BODY[5] BODY[6] BODY[7] BODY[8] BODY[9] BODY[10] BODY[11] BODY[13] BODY[14] BODY[15] BODY[16] BODY[21]) S: ... 4.3.4. User-Initiated Synchronization After the client has finished the main synchronization process as described in Sections 4.3.1-4.3.3, the user may optionally request additional synchronization steps while the client is still online. This is not any different from the process described in Sections 4.3.2 and 4.3.3. Typical examples are: 1) fetch all messages selected in UI. 2) fetch all messages marked as \Flagged on the server. 4.4. Special Case: Descriptor-Only Synchronization For some mailboxes, fetching the descriptors might be the entire synchronization step. Practical experience with IMAP has shown that a certain class of mailboxes (e.g., "archival" mailboxes) are used primarily for long-term storage of important messages that the human wants to have instantly available on demand but does not want cluttering up the disconnected client's cache at any other time. Messages in this kind of mailbox would be fetched exclusively by explicit actions queued by the local MUA. Thus, the only synchronization desirable on this kind of mailbox is fetching enough descriptor information for the user to be able to identify messages for subsequent download. Melnikov Informational [Page 22] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Special mailboxes that receive messages from a high volume, low priority mailing list might also be in this category, at least when the human is in a hurry. 4.5. Special Case: Fast New-Only Synchronization In some cases, the human might be in such a hurry that he or she doesn't care about changes to old messages, just about new messages. In this case, the client can skip the UID FETCH command that obtains the flags and UIDs for old messages (1:). 4.6. Special Case: Blind FETCH In some cases, the human may know (for whatever reason) that he or she always wants to fetch any new messages in a particular mailbox, unconditionally. In this case, the client can just fetch the messages themselves, rather than just the descriptors, by using a command like: tag1 UID FETCH :* (FLAGS BODY.PEEK[]) Note that this example ignores the fact that the messages can be arbitrary long. The disconnected client MUST always check for message size before downloading, unless explicitly told otherwise. A well-behaved client should instead use something like the following: 1) Issue "tag1 UID FETCH :* (FLAGS RFC822.SIZE)". 2) From the message sizes returned in step 1, construct UID set . 3) Issue "tag2 UID FETCH (BODY.PEEK[])". or 1) Issue "tag1 UID FETCH :* (FLAGS)". 2) Construct UID set from the responses of step 1. 3) Issue "tag2 SEARCH UID SMALLER ". Construct UID set from the result of the SEARCH command. 4) Issue "tag3 UID FETCH (BODY.PEEK[])". Melnikov Informational [Page 23] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 or 1) Issue "tag1 UID FETCH :* (FLAGS BODY.PEEK[]<0.>)", where should be replaced with the maximal message size the client is willing to download. Note: In response to such a command, the server will only return partial data if the message is longer than . It will return the full message data for any message whose size is smaller than or equal to . In the former case, the client will not be able to extract the full MIME structure of the message from the truncated data, so the client should include BODYSTRUCTURE in the UID FETCH command as well. 5. Implementation Considerations Below are listed some common implementation pitfalls that should be considered when implementing a disconnected client. 1) Implementing fake UIDs on the client. A message scheduled to be uploaded has no UID, as UIDs are selected by the server. The client may implement fake UIDs internally in order to reference not-yet-uploaded messages in further operations. (For example, a message could be scheduled to be uploaded, but subsequently marked as deleted or copied to another mailbox). Here, the client MUST NOT under any circumstances send these fake UIDs to the server. Also, client implementers should be reminded that according to [IMAP4] a UID is a 32-bit unsigned integer excluding 0. So, both 4294967295 and 2147483648 are valid UIDs, and 0 and -1 are both invalid. Some disconnected mail clients have been known to send negative numbers (e.g., "-1") as message UIDs to servers during synchronization. Situation 1: The user starts composing a new message, edits it, saves it, continues to edit it, and saves it again. A disconnected client may record in its replay log (log of operations to be replayed on the server during synchronization) the sequence of operations as shown below. For the purpose of this situation, we assume that all draft messages are stored in the mailbox called Drafts on an IMAP server. We will also use the following conventions: is the UID of the intermediate version of the draft when it was saved for the first time. This is a fake UID generated on the client. is the UID of the final version of the draft. This is another fake UID generated on the client. Melnikov Informational [Page 24] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 1) APPEND Drafts (\Seen $MDNSent \Drafts) {} ...first version of the message follows... 2) APPEND Drafts (\Seen $MDNSent \Drafts) {} ...final version of the message follows... 3) STORE +FLAGS (\Deleted) Step 1 corresponds to the first attempt to save the draft message, step 2 corresponds to the second attempt to save the draft message, and step 3 deletes the first version of the draft message saved in step 1. A naive disconnected client may send the command in step 3 without replacing the fake client generated with the value returned by the server in step 1. A server will probably reject this command, which will make the client believe that the synchronization sequence has failed. 2) Section 5.1 discusses common implementation errors related to error recovery during playback. 3) Don't assume that the disconnected client is the only client used by the user. Situation 2: Some clients may use the \Deleted flag as an indicator that the message should not appear in the user's view. Usage of the \Deleted flag for this purpose is not safe, as other clients (e.g., online clients) might EXPUNGE the mailbox at any time. 4) Beware of data dependencies between synchronization operations. It might be very tempting for a client writer to perform some optimizations on the playback log. Such optimizations might include removing redundant operations (for example, see optimization 2 in Section 5.3), or their reordering. It is not always safe to reorder or remove redundant operations during synchronization because some operations may have dependencies (as Situation 3 demonstrates). So, if in doubt, don't do this. Situation 3: The user copies a message out of a mailbox and then deletes the mailbox. Melnikov Informational [Page 25] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 C: A001 SELECT Old-Mail S: ... C: A002 UID COPY 111 ToDo S: A002 OK [COPYUID 1022843345 111 94] Copy completed ... C: A015 CLOSE S: A015 OK Completed C: A016 DELETE Old-Mail S: A016 OK Mailbox deletion completed successfully If the client performs DELETE (tag A016) first and COPY (tag A002) second, then the COPY fails. Also, the message that the user so carefully copied into another mailbox has been lost. 5.1. Error Recovery during Playback Error recovery during synchronization is one of the trickiest parts to get right. Below, we will discuss certain error conditions and suggest possible choices for handling them. 1) Lost connection to the server. The client MUST remember the current position in the playback (replay) log and replay it starting from the interrupted operation (the last command issued by the client, but not acknowledged by the server) the next time it successfully connects to the same server. If the connection was lost while executing a non- idempotent IMAP command (see the definition in Section 1), then when the client is reconnected, it MUST make sure that the interrupted command was indeed not executed. If it wasn't executed, the client must restart playback from the interrupted command, otherwise from the following command. Upon reconnect, care must be taken in order to properly reapply logical operations that are represented by multiple IMAP commands, e.g., UID EXPUNGE emulation when UID EXPUNGE is not supported by the server (see Section 4.2.4). Once the client detects that the connection to the server was lost, it MUST stop replaying its log. There are existing disconnected clients that, to the great annoyance of users, pop up an error dialog for each and every playback operation that fails. 2) Copying/appending messages to a mailbox that doesn't exist. (The server advertises this condition by sending the TRYCREATE response code in the tagged NO response to the APPEND or COPY command.) Melnikov Informational [Page 26] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 The user should be advised about the situation and be given one of the following choices: a) Try to recreate a mailbox. b) Copy/upload messages to another mailbox. c) Skip copy/upload. d) Abort replay. 3) Copying messages from a mailbox that doesn't exist, or renaming or getting/changing ACLs [ACL] on a mailbox that doesn't exist: a) Skip operation. b) Abort replay. 4) Deleting mailboxes or deleting/expunging messages that no longer exist. This is actually is not an error and should be ignored by the client. 5) Performing operations on messages that no longer exist. a) Skip operation. b) Abort replay. In the case of changing flags on an expunged message, the client should silently ignore the error. Note 1: Several synchronization operations map to multiple IMAP commands (for example, "move" described in 4.2.2). The client must guarantee atomicity of each such multistep operation. For example, when performing a "move" between two mailboxes on the same server, if the server is unable to copy messages, the client MUST NOT attempt to set the \Deleted flag on the messages being copied, let alone expunge them. However, the client MAY consider that move operation to have succeeded even if the server was unable to set the \Deleted flag on copied messages. Note 2: Many synchronization operations have data dependencies. A failed operation must cause all dependent operations to fail as well. The client should check this and MUST NOT try to perform all dependent operations blindly (unless the user corrected the original problem). For example, a message may be scheduled to be appended to a mailbox on the server and later on the appended message may be copied to another mailbox. If the APPEND operation fails, the client must not attempt to COPY the failed message later on. (See also Section 5, Situation 3). Melnikov Informational [Page 27] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 5.2. Quality of Implementation Issues Below, some quality of implementation issues are listed for disconnected clients. They will help to write a disconnected client that works correctly, performs synchronization as quickly as possible (and thus can make the user happier as well as save her some money), and minimizes the server load: 1) Don't lose information. No matter how smart your client is in other areas, if it loses information, users will get very upset. 2) Don't do work unless explicitly asked. Be flexible. Ask all questions BEFORE starting synchronization, if possible. 3) Minimize traffic. The client MUST NOT issue a command if the client already received the required information from the server. The client MUST make use of UIDPLUS extension if it is supported by the server. See also optimization 1 in Section 5.3. 4) Minimize the number of round-trips. Round-trips kill performance, especially on links with high latency. Sections 4.2.2.5 and 5.2 give some advice on how to minimize the number of round-trips. See also optimization 1 in Section 5.3. 5.3. Optimizations Some useful optimizations are described in this section. A disconnected client that supports the recommendations listed below will give the user a more pleasant experience. 1) The initial OK or PREAUTH responses may contain the CAPABILITY response code as described in Section 7.1 of [IMAP4]. This response code gives the same information as returned by the CAPABILITY command*. A disconnected client that pays attention to this response code can avoid sending CAPABILITY command and will save a round-trip. Melnikov Informational [Page 28] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 * Note: Some servers report in the CAPABILITY response code extensions that are only relevant in unauthenticated state or in all states. Such servers usually send another CAPABILITY response code upon successful authentication using LOGIN or AUTHENTICATE command (that negotiates no security layer; see Section 6.2.2 of [IMAP4]). The CAPABILITY response code sent upon successful LOGIN/AUTHENTICATE might be different from the CAPABILITY response code in the initial OK response, as extensions only relevant for unauthenticated state will not be advertised, and some additional extensions available only in authenticated and/or selected state will be. Example 9: S: * OK [CAPABILITY IMAP4REV1 LOGIN-REFERRALS STARTTLS AUTH=DIGEST-MD5 AUTH=SRP] imap.example.com ready C: 2 authenticate DIGEST-MD5 S: 2 OK [CAPABILITY IMAP4REV1 IDLE NAMESPACE MAILBOX-REFERRALS SCAN SORT THREAD=REFERENCES THREAD=ORDEREDSUBJECT MULTIAPPEND] User authenticated (no layer) 2) An advanced disconnected client may choose to optimize its replay log. For example, there might be some operations that are redundant (the list is not complete): a) an EXPUNGE followed by another EXPUNGE or CLOSE; b) changing flags (other than the \Deleted flag) on a message that gets immediately expunged; c) opening and closing the same mailbox. When optimizing, be careful about data dependencies between commands. For example, if the client is wishing to optimize (see case b, above) tag1 UID STORE +FLAGS (\Deleted) ... tag2 UID STORE +FLAGS (\Flagged) ... tag3 UID COPY "Backup" ... tag4 UID EXPUNGE it can't remove the second UID STORE command because the message is being copied before it gets expunged. In general, it might be a good idea to keep mailboxes open during synchronization (see case c above), if possible. This can be more easily achieved in conjunction with optimization 3 described below. Melnikov Informational [Page 29] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 3) Perform some synchronization steps in parallel, if possible. Several synchronization steps don't depend on each other and thus can be performed in parallel. Because the server machine is usually more powerful than the client machine and can perform some operations in parallel, this may speed up the total time of synchronization. In order to achieve such parallelization, the client will have to open more than one connection to the same server. Client writers should not forget about non-trivial cost associated with establishing a TCP connection and performing an authentication. The disconnected client MUST NOT use one connection per mailbox. In most cases, it is sufficient to have two connections. The disconnected client SHOULD avoid selecting the same mailbox in more than one connection; see Section 3.1.1 of [RFC2683] for more details. Any mailbox synchronization MUST start with checking the UIDVALIDITY as described in Section 4.1 of this document. The client MAY use STATUS command to check UID Validity of a non- selected mailbox. This is preferable to opening many connections to the same server to perform synchronization of multiple mailboxes simultaneously. As described in Section 5.3.10 of [IMAP4], this SHOULD NOT be used on the selected mailbox. 6. IMAP Extensions That May Help The following extensions can save traffic and/or the number of round-trips: 1) The use of [UIDPLUS] is discussed in Sections 4.1, 4.2.1, 4.2.2.1 and 4.2.4. 2) The use of the MULTIAPPEND and LITERAL+ extensions for uploading messages is discussed in Section 4.2.2.5. 3) Use the CONDSTORE extension (see Section 6.1) for quick flag resynchronization. 6.1. CONDSTORE Extension An advanced disconnected mail client should use the [CONDSTORE] extension when it is supported by the server. The client must cache the value from HIGHESTMODSEQ OK response code received on mailbox opening and update it whenever the server sends MODSEQ FETCH data items. Melnikov Informational [Page 30] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 If the client receives NOMODSEQ OK untagged response instead of HIGHESTMODSEQ, it MUST remove the last known HIGHESTMODSEQ value from its cache and follow the more general instructions in Section 3. When the client opens the mailbox for synchronization, it first compares UIDVALIDITY as described in step d-1 in Section 3. If the cached UIDVALIDITY value matches the one returned by the server, the client MUST compare the cached value of HIGHESTMODSEQ with the one returned by the server. If the cached HIGHESTMODSEQ value also matches the one returned by the server, then the client MUST NOT fetch flags for cached messages, as they hasn't changed. If the value on the server is higher than the cached one, the client MAY use "SEARCH MODSEQ " to find all messages with flags changed since the last time the client was online and had the mailbox opened. Alternatively, the client MAY use "FETCH 1:* (FLAGS) (CHANGEDSINCE )". The latter operation combines searching for changed messages and fetching new information. In all cases, the client still needs to fetch information about new messages (if requested by the user) as well as discover which messages have been expunged. Step d ("Server-to-client synchronization") in Section 4 in the presence of the CONDSTORE extension is amended as follows: d) "Server-to-client synchronization" - For each mailbox that requires synchronization, do the following: 1a) Check the mailbox UIDVALIDITY (see section 4.1 for more details) with SELECT/EXAMINE/STATUS. If the UIDVALIDITY value returned by the server differs, the client MUST * empty the local cache of that mailbox; * "forget" the cached HIGHESTMODSEQ value for the mailbox; * remove any pending "actions" that refer to UIDs in that mailbox (note that this doesn't affect actions performed on client-generated fake UIDs; see Section 5); and * skip steps 1b and 2-II; 1b) Check the mailbox HIGHESTMODSEQ. If the cached value is the same as the one returned by the server, skip fetching message flags on step 2-II, i.e., the client only has to find out which messages got expunged. Melnikov Informational [Page 31] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 2) Fetch the current "descriptors". I) Discover new messages. II) Discover changes to old messages and flags for new messages using "FETCH 1:* (FLAGS) (CHANGEDSINCE )" or "SEARCH MODSEQ ". Discover expunged messages; for example, using "UID SEARCH 1:". (All messages not returned in this command are expunged.) 3) Fetch the bodies of any "interesting" messages that the client doesn't already have. Example 10: The UIDVALIDITY value is the same, but the HIGHESTMODSEQ value has changed on the server while the client was offline. C: A142 SELECT INBOX S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 201] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: * OK [HIGHESTMODSEQ 20010715194045007] S: A142 OK [READ-WRITE] SELECT completed After that, either: C: A143 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 20010715194032001) S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) FLAGS (\Deleted)) S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) FLAGS ($NoJunk $AutoJunk $MDNSent)) ... S: A143 OK FETCH completed or: Melnikov Informational [Page 32] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 C: A143 UID SEARCH MODSEQ 20010715194032001 UID 1:20 S: * SEARCH 6 9 11 12 18 19 20 23 (MODSEQ 20010917162500) S: A143 OK Search complete C: A144 UID SEARCH 1:20 S: * SEARCH 6 9 ... S: A144 OK FETCH completed 7. Security Considerations It is believed that this document does not raise any new security concerns that are not already present in the base [IMAP4] protocol, and these issues are discussed in [IMAP4]. Additional security considerations may be found in different extensions mentioned in this document; in particular, in [UIDPLUS], [LITERAL+], [CONDSTORE], [MULTIAPPEND], and [UNSELECT]. Implementers are also reminded about the importance of thorough testing. 8. References 8.1. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS extension", RFC 4315, December 2005. [LITERAL+] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, January 1997. [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. [MULTIAPPEND] Crispin, M., "Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension", RFC 3502, March 2003. [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) UNSELECT command", RFC 3691, February 2004. [RFC2683] Leiba, B., "IMAP4 Implementation Recommendations", RFC 2683, September 1999. Melnikov Informational [Page 33] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 8.2. Informative References [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [IMAP-MODEL] Crispin, M., "Distributed Electronic Mail Models in IMAP4", RFC 1733, December 1994. 9. Acknowledgements This document is based on version 01 of the text written by Rob Austein in November 1994. The editor appreciates comments posted by Mark Crispin to the IMAP mailing list and the comments/corrections/ideas received from Grant Baillie, Cyrus Daboo, John G. Myers, Chris Newman, and Timo Sirainen. The editor would also like to thank the developers of Netscape Messenger and Mozilla mail clients for providing examples of disconnected mail clients that served as a base for many recommendations in this document. Editor's Address Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX United Kingdom Phone: +44 77 53759732 EMail: alexey.melnikov@isode.com Melnikov Informational [Page 34] RFC 4549 Synch Ops for Disconnected IMAP4 Clients June 2006 Full Copyright Statement Copyright (C) The Internet Society (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is provided by the IETF Administrative Support Activity (IASA). Melnikov Informational [Page 35] ================================================ FILE: docs/rfcs/rfc4551.IMAP_Conditional_STORE_or_Quick_flag_changes_resync.txt ================================================ Network Working Group A. Melnikov Request for Comments: 4551 Isode Ltd. Updates: 3501 S. Hole Category: Standards Track ACI WorldWide/MessagingDirect June 2006 IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The Internet Society (2006). Abstract Often, multiple IMAP (RFC 3501) clients need to coordinate changes to a common IMAP mailbox. Examples include different clients working on behalf of the same user, and multiple users accessing shared mailboxes. These clients need a mechanism to synchronize state changes for messages within the mailbox. They must be able to guarantee that only one client can change message state (e.g., message flags) at any time. An example of such an application is use of an IMAP mailbox as a message queue with multiple dequeueing clients. The Conditional Store facility provides a protected update mechanism for message state information that can detect and resolve conflicts between multiple writing mail clients. The Conditional Store facility also allows a client to quickly resynchronize mailbox flag changes. This document defines an extension to IMAP (RFC 3501). Melnikov & Hole Standards Track [Page 1] RFC 4551 IMAP Extension for Conditional STORE June 2006 Table of Contents 1. Introduction and Overview ................................. 3 2. Conventions Used in This Document ......................... 5 3. IMAP Protocol Changes ..................................... 6 3.1. New OK untagged responses for SELECT and EXAMINE ......... 6 3.1.1. HIGHESTMODSEQ response code ............................ 6 3.1.2. NOMODSEQ response code ................................. 7 3.2. STORE and UID STORE Commands ............................. 7 3.3 FETCH and UID FETCH Commands ..............................13 3.3.1. CHANGEDSINCE FETCH modifier ............................13 3.3.2. MODSEQ message data item in FETCH Command ..............14 3.4. MODSEQ search criterion in SEARCH ........................16 3.5. Modified SEARCH untagged response ........................17 3.6. HIGHESTMODSEQ status data items ..........................17 3.7. CONDSTORE parameter to SELECT and EXAMINE ................18 3.8. Additional quality of implementation issues ..............18 4. Formal Syntax .............................................19 5. Server implementation considerations ......................21 6. Security Considerations ...................................22 7. IANA Considerations .......................................22 8. References ................................................23 8.1. Normative References .....................................23 8.2. Informative References ...................................23 9. Acknowledgements ..........................................23 Melnikov & Hole Standards Track [Page 2] RFC 4551 IMAP Extension for Conditional STORE June 2006 1. Introduction and Overview The Conditional STORE extension is present in any IMAP4 implementation that returns "CONDSTORE" as one of the supported capabilities in the CAPABILITY command response. An IMAP server that supports this extension MUST associate a positive unsigned 64-bit value called a modification sequence (mod-sequence) with every IMAP message. This is an opaque value updated by the server whenever a metadata item is modified. The server MUST guarantee that each STORE command performed on the same mailbox (including simultaneous stores to different metadata items from different connections) will get a different mod-sequence value. Also, for any two successful STORE operations performed in the same session on the same mailbox, the mod-sequence of the second completed operation MUST be greater than the mod-sequence of the first completed. Note that the latter rule disallows the use of the system clock as a mod-sequence, because if system time changes (e.g., an NTP [NTP] client adjusting the time), the next generated value might be less than the previous one. Mod-sequences allow a client that supports the CONDSTORE extension to determine if a message metadata has changed since some known moment. Whenever the state of a flag changes (i.e., the flag is added where previously it wasn't set, or the flag is removed and before it was set) the value of the modification sequence for the message MUST be updated. Adding the flag when it is already present or removing when it is not present SHOULD NOT change the mod-sequence. When a message is appended to a mailbox (via the IMAP APPEND command, COPY to the mailbox, or using an external mechanism) the server generates a new modification sequence that is higher than the highest modification sequence of all messages in the mailbox and assigns it to the appended message. The server MAY store separate (per-message) modification sequence values for different metadata items. If the server does so, per- message mod-sequence is the highest mod-sequence of all metadata items for the specified message. The server that supports this extension is not required to be able to store mod-sequences for every available mailbox. Section 3.1.2 describes how the server may act if a particular mailbox doesn't support the persistent storage of mod-sequences. Melnikov & Hole Standards Track [Page 3] RFC 4551 IMAP Extension for Conditional STORE June 2006 This extension makes the following changes to the IMAP4 protocol: a) adds UNCHANGEDSINCE STORE modifier. b) adds the MODIFIED response code which should be used with an OK response to the STORE command. (It can also be used in a NO response.) c) adds a new MODSEQ message data item for use with the FETCH command. d) adds CHANGEDSINCE FETCH modifier. e) adds a new MODSEQ search criterion. f) extends the syntax of untagged SEARCH responses to include mod-sequence. g) adds new OK untagged responses for the SELECT and EXAMINE commands. h) defines an additional parameter to SELECT/EXAMINE commands. i) adds the HIGHESTMODSEQ status data item to the STATUS command. A client supporting CONDSTORE extension indicates its willingness to receive mod-sequence updates in all untagged FETCH responses by issuing: - a SELECT or EXAMINE command with the CONDSTORE parameter, - a STATUS (HIGHESTMODSEQ) command, - a FETCH or SEARCH command that includes the MODSEQ message data item, - a FETCH command with the CHANGEDSINCE modifier, or - a STORE command with the UNCHANGEDSINCE modifier. The server MUST include mod-sequence data in all subsequent untagged FETCH responses (until the connection is closed), whether they were caused by a regular STORE, a STORE with UNCHANGEDSINCE modifier, or an external agent. This document uses the term "CONDSTORE-aware client" to refer to a client that announces its willingness to receive mod-sequence updates as described above. The term "CONDSTORE enabling command" will refer any of the commands listed above. A future extension to this document may extend the list of CONDSTORE enabling commands. A first CONDSTORE enabling command executed in the session MUST cause the Melnikov & Hole Standards Track [Page 4] RFC 4551 IMAP Extension for Conditional STORE June 2006 server to return HIGHESTMODSEQ (Section 3.1.1) unless the server has sent NOMODSEQ (Section 3.1.2) response code when the currently selected mailbox was selected. The rest of this document describes the protocol changes more rigorously. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [KEYWORDS]. In examples, lines beginning with "S:" are sent by the IMAP server, and lines beginning with "C:" are sent by the client. Line breaks may appear in example commands solely for editorial clarity; when present in the actual message, they are represented by "CRLF". Formal syntax is defined using ABNF [ABNF]. The term "metadata" or "metadata item" is used throughout this document. It refers to any system or user-defined keyword. Future documents may extend "metadata" to include other dynamic message data. Some IMAP mailboxes are private, accessible only to the owning user. Other mailboxes are not, either because the owner has set an Access Control List [ACL] that permits access by other users, or because it is a shared mailbox. Let's call a metadata item "shared" for the mailbox if any changes to the metadata items are persistent and visible to all other users accessing the mailbox. Otherwise, the metadata item is called "private". Note that private metadata items are still visible to all sessions accessing the mailbox as the same user. Also note that different mailboxes may have different metadata items as shared. See Section 1 for the definition of a "CONDSTORE-aware client" and a "CONDSTORE enabling command". Melnikov & Hole Standards Track [Page 5] RFC 4551 IMAP Extension for Conditional STORE June 2006 3. IMAP Protocol Changes 3.1. New OK Untagged Responses for SELECT and EXAMINE This document adds two new response codes, HIGHESTMODSEQ and NOMODSEQ. One of those response codes MUST be returned in the OK untagged response for a successful SELECT/EXAMINE command. When opening a mailbox, the server must check if the mailbox supports the persistent storage of mod-sequences. If the mailbox supports the persistent storage of mod-sequences and the mailbox open operation succeeds, the server MUST send the OK untagged response including HIGHESTMODSEQ response code. If the persistent storage for the mailbox is not supported, the server MUST send the OK untagged response including NOMODSEQ response code instead. 3.1.1. HIGHESTMODSEQ Response Code This document adds a new response code that is returned in the OK untagged response for the SELECT and EXAMINE commands. A server supporting the persistent storage of mod-sequences for the mailbox MUST send the OK untagged response including HIGHESTMODSEQ response code with every successful SELECT or EXAMINE command: OK [HIGHESTMODSEQ ] where is the highest mod-sequence value of all messages in the mailbox. When the server changes UIDVALIDITY for a mailbox, it doesn't have to keep the same HIGHESTMODSEQ for the mailbox. A disconnected client can use the value of HIGHESTMODSEQ to check if it has to refetch metadata from the server. If the UIDVALIDITY value has changed for the selected mailbox, the client MUST delete the cached value of HIGHESTMODSEQ. If UIDVALIDITY for the mailbox is the same, and if the HIGHESTMODSEQ value stored in the client's cache is less than the value returned by the server, then some metadata items on the server have changed since the last synchronization, and the client needs to update its cache. The client MAY use SEARCH MODSEQ (Section 3.4) to find out exactly which metadata items have changed. Alternatively, the client MAY issue FETCH with the CHANGEDSINCE modifier (Section 3.3.1) in order to fetch data for all messages that have metadata items changed since some known modification sequence. Example 1: C: A142 SELECT INBOX S: * 172 EXISTS Melnikov & Hole Standards Track [Page 6] RFC 4551 IMAP Extension for Conditional STORE June 2006 S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: * OK [HIGHESTMODSEQ 715194045007] S: A142 OK [READ-WRITE] SELECT completed 3.1.2. NOMODSEQ Response Code A server that doesn't support the persistent storage of mod-sequences for the mailbox MUST send the OK untagged response including NOMODSEQ response code with every successful SELECT or EXAMINE command. A server that returned NOMODSEQ response code for a mailbox, which subsequently receives one of the following commands while the mailbox is selected: - a FETCH command with the CHANGEDSINCE modifier, - a FETCH or SEARCH command that includes the MODSEQ message data item, or - a STORE command with the UNCHANGEDSINCE modifier MUST reject any such command with the tagged BAD response. Example 2: C: A142 SELECT INBOX S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: * OK [NOMODSEQ] Sorry, this mailbox format doesn't support modsequences S: A142 OK [READ-WRITE] SELECT completed 3.2. STORE and UID STORE Commands This document defines the following STORE modifier (see Section 2.5 of [IMAPABNF]): UNCHANGEDSINCE For each message specified in the message set, the server performs the following. If the mod-sequence of any metadata item of the Melnikov & Hole Standards Track [Page 7] RFC 4551 IMAP Extension for Conditional STORE June 2006 message is equal or less than the specified UNCHANGEDSINCE value, then the requested operation (as described by the message data item) is performed. If the operation is successful, the server MUST update the mod-sequence attribute of the message. An untagged FETCH response MUST be sent, even if the .SILENT suffix is specified, and the response MUST include the MODSEQ message data item. This is required to update the client's cache with the correct mod-sequence values. See Section 3.3.2 for more details. However, if the mod-sequence of any metadata item of the message is greater than the specified UNCHANGEDSINCE value, then the requested operation MUST NOT be performed. In this case, the mod-sequence attribute of the message is not updated, and the message number (or unique identifier in the case of the UID STORE command) is added to the list of messages that failed the UNCHANGESINCE test. When the server finished performing the operation on all the messages in the message set, it checks for a non-empty list of messages that failed the UNCHANGESINCE test. If this list is non-empty, the server MUST return in the tagged response a MODIFIED response code. The MODIFIED response code includes the message set (for STORE) or set of UIDs (for UID STORE) of all messages that failed the UNCHANGESINCE test. Example 3: All messages pass the UNCHANGESINCE test. C: a103 UID STORE 6,4,8 (UNCHANGEDSINCE 12121230045) +FLAGS.SILENT (\Deleted) S: * 1 FETCH (UID 4 MODSEQ (12121231000)) S: * 2 FETCH (UID 6 MODSEQ (12121230852)) S: * 4 FETCH (UID 8 MODSEQ (12121130956)) S: a103 OK Conditional Store completed Example 4: C: a104 STORE * (UNCHANGEDSINCE 12121230045) +FLAGS.SILENT (\Deleted $Processed) S: * 50 FETCH (MODSEQ (12111230047)) S: a104 OK Store (conditional) completed Example 5: C: c101 STORE 1 (UNCHANGEDSINCE 12121230045) -FLAGS.SILENT (\Deleted) S: * OK [HIGHESTMODSEQ 12111230047] Melnikov & Hole Standards Track [Page 8] RFC 4551 IMAP Extension for Conditional STORE June 2006 S: * 50 FETCH (MODSEQ (12111230048)) S: c101 OK Store (conditional) completed HIGHESTMODSEQ response code was sent by the server presumably because this was the first CONDSTORE enabling command. Example 6: In spite of the failure of the conditional STORE operation for message 7, the server continues to process the conditional STORE in order to find all messages that fail the test. C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) +FLAGS.SILENT (\Deleted) S: * 5 FETCH (MODSEQ (320162350)) S: d105 OK [MODIFIED 7,9] Conditional STORE failed Example 7: Same as above, but the server follows the SHOULD recommendation in Section 6.4.6 of [IMAP4]. C: d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) +FLAGS.SILENT (\Deleted) S: * 7 FETCH (MODSEQ (320162342) FLAGS (\Seen \Deleted)) S: * 5 FETCH (MODSEQ (320162350)) S: * 9 FETCH (MODSEQ (320162349) FLAGS (\Answered)) S: d105 OK [MODIFIED 7,9] Conditional STORE failed Use of UNCHANGEDSINCE with a modification sequence of 0 always fails if the metadata item exists. A system flag MUST always be considered existent, whether it was set or not. Example 8: C: a102 STORE 12 (UNCHANGEDSINCE 0) +FLAGS.SILENT ($MDNSent) S: a102 OK [MODIFIED 12] Conditional STORE failed The client has tested the presence of the $MDNSent user-defined keyword. Note: A client trying to make an atomic change to the state of a particular metadata item (or a set of metadata items) should be prepared to deal with the case when the server returns the MODIFIED response code if the state of the metadata item being watched hasn't changed (but the state of some other metadata item has). This is necessary, because some servers don't store separate mod-sequences Melnikov & Hole Standards Track [Page 9] RFC 4551 IMAP Extension for Conditional STORE June 2006 for different metadata items. However, a server implementation SHOULD avoid generating spurious MODIFIED responses for +FLAGS/-FLAGS STORE operations, even when the server stores a single mod-sequence per message. Section 5 describes how this can be achieved. Unless the server has included an unsolicited FETCH to update client's knowledge about messages that have failed the UNCHANGEDSINCE test, upon receipt of the MODIFIED response code, the client SHOULD try to figure out if the required metadata items have indeed changed by issuing FETCH or NOOP command. It is RECOMMENDED that the server avoids the need for the client to do that by sending an unsolicited FETCH response (Examples 9 and 10). If the required metadata items haven't changed, the client SHOULD retry the command with the new mod-sequence. The client SHOULD allow for a configurable but reasonable number of retries (at least 2). Example 9: In the example below, the server returns the MODIFIED response code without sending information describing why the STORE UNCHANGEDSINCE operation has failed. C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) +FLAGS.SILENT ($Processed) S: * 100 FETCH (MODSEQ (303181230852)) S: * 102 FETCH (MODSEQ (303181230852)) ... S: * 150 FETCH (MODSEQ (303181230852)) S: a106 OK [MODIFIED 101] Conditional STORE failed The flag $Processed was set on the message 101... C: a107 NOOP S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) S: a107 OK Or the flag hasn't changed, but another has (note that this server behaviour is discouraged. Server implementers should also see Section 5)... C: b107 NOOP S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) S: b107 OK ...and the client retries the operation for the message 101 with the updated UNCHANGEDSINCE value Melnikov & Hole Standards Track [Page 10] RFC 4551 IMAP Extension for Conditional STORE June 2006 C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) +FLAGS.SILENT ($Processed) S: * 101 FETCH (MODSEQ (303181230852)) S: b108 OK Conditional Store completed Example 10: Same as above, but the server avoids the need for the client to poll for changes. The flag $Processed was set on the message 101 by another client... C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) +FLAGS.SILENT ($Processed) S: * 100 FETCH (MODSEQ (303181230852)) S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed)) S: * 102 FETCH (MODSEQ (303181230852)) ... S: * 150 FETCH (MODSEQ (303181230852)) S: a106 OK [MODIFIED 101] Conditional STORE failed Or the flag hasn't changed, but another has (note that this server behaviour is discouraged. Server implementers should also see Section 5)... C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) +FLAGS.SILENT ($Processed) S: * 100 FETCH (MODSEQ (303181230852)) S: * 101 FETCH (MODSEQ (303011130956) FLAGS (\Deleted \Answered)) S: * 102 FETCH (MODSEQ (303181230852)) ... S: * 150 FETCH (MODSEQ (303181230852)) S: a106 OK [MODIFIED 101] Conditional STORE failed ...and the client retries the operation for the message 101 with the updated UNCHANGEDSINCE value C: b108 STORE 101 (UNCHANGEDSINCE 303011130956) +FLAGS.SILENT ($Processed) S: * 101 FETCH (MODSEQ (303181230852)) S: b108 OK Conditional Store completed Or the flag hasn't changed, but another has (nice server behaviour. Server implementers should also see Section 5)... C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) +FLAGS.SILENT ($Processed) Melnikov & Hole Standards Track [Page 11] RFC 4551 IMAP Extension for Conditional STORE June 2006 S: * 100 FETCH (MODSEQ (303181230852)) S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed \Deleted \Answered)) S: * 102 FETCH (MODSEQ (303181230852)) ... S: * 150 FETCH (MODSEQ (303181230852)) S: a106 OK Conditional STORE completed Example 11: The following example is based on the example from the Section 4.2.3 of [RFC-2180] and demonstrates that the MODIFIED response code may be also returned in the tagged NO response. Client tries to conditionally STORE flags on a mixture of expunged and non-expunged messages; one message fails the UNCHANGEDSINCE test. C: B001 STORE 1:7 (UNCHANGEDSINCE 320172338) +FLAGS (\SEEN) S: * 1 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) S: * 3 FETCH (MODSEQ (320172342) FLAGS (\SEEN)) S: B001 NO [MODIFIED 2] Some of the messages no longer exist. C: B002 NOOP S: * 4 EXPUNGE S: * 4 EXPUNGE S: * 4 EXPUNGE S: * 4 EXPUNGE S: * 2 FETCH (MODSEQ (320172340) FLAGS (\Deleted \Answered)) S: B002 OK NOOP Completed. By receiving FETCH responses for messages 1 and 3, and EXPUNGE responses that indicate that messages 4 through 7 have been expunged, the client retries the operation only for the message 2. The updated UNCHANGEDSINCE value is used. C: b003 STORE 2 (UNCHANGEDSINCE 320172340) +FLAGS (\Seen) S: * 2 FETCH (MODSEQ (320180050)) S: b003 OK Conditional Store completed Note: If a message is specified multiple times in the message set, and the server doesn't internally eliminate duplicates from the message set, it MUST NOT fail the conditional STORE operation for the second (or subsequent) occurrence of the message if the operation completed successfully for the first occurrence. For example, if the client specifies: Melnikov & Hole Standards Track [Page 12] RFC 4551 IMAP Extension for Conditional STORE June 2006 e105 STORE 7,3:9 (UNCHANGEDSINCE 12121230045) +FLAGS.SILENT (\Deleted) the server must not fail the operation for message 7 as part of processing "3:9" if it succeeded when message 7 was processed the first time. Once the client specified the UNCHANGEDSINCE modifier in a STORE command, the server MUST include the MODSEQ fetch response data items in all subsequent unsolicited FETCH responses. This document also changes the behaviour of the server when it has performed a STORE or UID STORE command and the UNCHANGEDSINCE modifier is not specified. If the operation is successful for a message, the server MUST update the mod-sequence attribute of the message. The server is REQUIRED to include the mod-sequence value whenever it decides to send the unsolicited FETCH response to all CONDSTORE-aware clients that have opened the mailbox containing the message. Server implementers should also see Section 3.8 for additional quality of implementation issues related to the STORE command. 3.3. FETCH and UID FETCH Commands 3.3.1. CHANGEDSINCE FETCH Modifier This document defines the following FETCH modifier (see Section 2.4 of [IMAPABNF]): CHANGEDSINCE CHANGEDSINCE FETCH modifier allows to create a further subset of the list of messages described by sequence set. The information described by message data items is only returned for messages that have mod-sequence bigger than . When CHANGEDSINCE FETCH modifier is specified, it implicitly adds MODSEQ FETCH message data item (Section 3.3.2). Example 12: C: s100 UID FETCH 1:* (FLAGS) (CHANGEDSINCE 12345) S: * 1 FETCH (UID 4 MODSEQ (65402) FLAGS (\Seen)) S: * 2 FETCH (UID 6 MODSEQ (75403) FLAGS (\Deleted)) S: * 4 FETCH (UID 8 MODSEQ (29738) FLAGS ($NoJunk $AutoJunk $MDNSent)) S: s100 OK FETCH completed Melnikov & Hole Standards Track [Page 13] RFC 4551 IMAP Extension for Conditional STORE June 2006 3.3.2. MODSEQ Message Data Item in FETCH Command This extension adds a MODSEQ message data item to the FETCH command. The MODSEQ message data item allows clients to retrieve mod-sequence values for a range of messages in the currently selected mailbox. Once the client specified the MODSEQ message data item in a FETCH request, the server MUST include the MODSEQ fetch response data items in all subsequent unsolicited FETCH responses. Syntax: MODSEQ The MODSEQ message data item causes the server to return MODSEQ fetch response data items. Syntax: MODSEQ ( ) MODSEQ response data items contain per-message mod-sequences. The MODSEQ response data item is returned if the client issued FETCH with MODSEQ message data item. It also allows the server to notify the client about mod-sequence changes caused by conditional STOREs (Section 3.2) and/or changes caused by external sources. Example 13: C: a FETCH 1:3 (MODSEQ) S: * 1 FETCH (MODSEQ (624140003)) S: * 2 FETCH (MODSEQ (624140007)) S: * 3 FETCH (MODSEQ (624140005)) S: a OK Fetch complete In this example, the client requests per-message mod-sequences for a set of messages. When a flag for a message is modified in a different session, the server sends an unsolicited FETCH response containing the mod- sequence for the message. Example 14: (Session 1, authenticated as a user "alex"). The user adds a shared flag \Deleted: C: A142 SELECT INBOX ... S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Answered \Deleted \Seen \*)] Limited Melnikov & Hole Standards Track [Page 14] RFC 4551 IMAP Extension for Conditional STORE June 2006 ... C: A160 STORE 7 +FLAGS.SILENT (\Deleted) S: * 7 FETCH (MODSEQ (2121231000)) S: A160 OK Store completed (Session 2, also authenticated as the user "alex"). Any changes to flags are always reported to all sessions authenticated as the same user as in the session 1. C: C180 NOOP S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) S: C180 OK Noop completed (Session 3, authenticated as a user "andrew"). As \Deleted is a shared flag, changes in session 1 are also reported in session 3: C: D210 NOOP S: * 7 FETCH (FLAGS (\Deleted \Answered) MODSEQ (12121231000)) S: D210 OK Noop completed The user modifies a private flag \Seen in session 1... C: A240 STORE 7 +FLAGS.SILENT (\Seen) S: * 7 FETCH (MODSEQ (12121231777)) S: A240 OK Store completed ...which is only reported in session 2... C: C270 NOOP S: * 7 FETCH (FLAGS (\Deleted \Answered \Seen) MODSEQ (12121231777)) S: C270 OK Noop completed ...but not in session 3. C: D300 NOOP S: D300 OK Noop completed And finally, the user removes flags \Answered (shared) and \Seen (private) in session 1. C: A330 STORE 7 -FLAGS.SILENT (\Answered \Seen) S: * 7 FETCH (MODSEQ (12121245160)) S: A330 OK Store completed Melnikov & Hole Standards Track [Page 15] RFC 4551 IMAP Extension for Conditional STORE June 2006 Both changes are reported in the session 2... C: C360 NOOP S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) S: C360 OK Noop completed ...and only changes to shared flags are reported in session 3. C: D390 NOOP S: * 7 FETCH (FLAGS (\Deleted) MODSEQ (12121245160)) S: D390 OK Noop completed Server implementers should also see Section 3.8 for additional quality of implementation issues related to the FETCH command. 3.4. MODSEQ Search Criterion in SEARCH The MODSEQ criterion for the SEARCH command allows a client to search for the metadata items that were modified since a specified moment. Syntax: MODSEQ [ ] Messages that have modification values that are equal to or greater than . This allows a client, for example, to find out which messages contain metadata items that have changed since the last time it updated its disconnected cache. The client may also specify (name of metadata item) and (type of metadata item) before . can be one of "shared", "priv" (private), or "all". The latter means that the server should use the biggest value among "priv" and "shared" mod- sequences for the metadata item. If the server doesn't store internally separate mod-sequences for different metadata items, it MUST ignore and . Otherwise, the server should use them to narrow down the search. For a flag , the corresponding has a form "/flags/" as defined in [IMAPABNF]. Note that the leading "\" character that denotes a system flag has to be escaped as per Section 4.3 of [IMAP4], as the uses syntax for quoted strings. If client specifies a MODSEQ criterion in a SEARCH command and the server returns a non-empty SEARCH result, the server MUST also append (to the end of the untagged SEARCH response) the highest mod-sequence for all messages being returned. See also Section 3.5. Melnikov & Hole Standards Track [Page 16] RFC 4551 IMAP Extension for Conditional STORE June 2006 Example 15: C: a SEARCH MODSEQ "/flags/\\draft" all 620162338 S: * SEARCH 2 5 6 7 11 12 18 19 20 23 (MODSEQ 917162500) S: a OK Search complete In the above example, the message numbers of any messages containing the string "IMAP4" in the "value" attribute of the "/comment" entry and having a mod-sequence equal to or greater than 620162338 for the "\Draft" flag are returned in the search results. Example 16: C: t SEARCH OR NOT MODSEQ 720162338 LARGER 50000 S: * SEARCH S: t OK Search complete, nothing found 3.5. Modified SEARCH Untagged Response Data: zero or more numbers mod-sequence value (omitted if no match) This document extends syntax of the untagged SEARCH response to include the highest mod-sequence for all messages being returned. If a client specifies a MODSEQ criterion in a SEARCH (or UID SEARCH) command and the server returns a non-empty SEARCH result, the server MUST also append (to the end of the untagged SEARCH response) the highest mod-sequence for all messages being returned. See Section 3.4 for examples. 3.6. HIGHESTMODSEQ Status Data Items This document defines a new status data item: HIGHESTMODSEQ The highest mod-sequence value of all messages in the mailbox. This is the same value that is returned by the server in the HIGHESTMODSEQ response code in an OK untagged response (see Section 3.1.1). If the server doesn't support the persistent storage of mod-sequences for the mailbox (see Section 3.1.2), the server MUST return 0 as the value of HIGHESTMODSEQ status data item. Melnikov & Hole Standards Track [Page 17] RFC 4551 IMAP Extension for Conditional STORE June 2006 Example 17: C: A042 STATUS blurdybloop (UIDNEXT MESSAGES HIGHESTMODSEQ) S: * STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292 HIGHESTMODSEQ 7011231777) S: A042 OK STATUS completed 3.7. CONDSTORE Parameter to SELECT and EXAMINE The CONDSTORE extension defines a single optional select parameter, "CONDSTORE", which tells the server that it MUST include the MODSEQ fetch response data items in all subsequent unsolicited FETCH responses. The CONDSTORE parameter to SELECT/EXAMINE helps avoid a race condition that might arise when one or more metadata items are modified in another session after the server has sent the HIGHESTMODSEQ response code and before the client was able to issue a CONDSTORE enabling command. Example 18: C: A142 SELECT INBOX (CONDSTORE) S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 4392] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: * OK [HIGHESTMODSEQ 715194045007] S: A142 OK [READ-WRITE] SELECT completed, CONDSTORE is now enabled 3.8. Additional Quality-of-Implementation Issues Server implementations should follow the following rule, which applies to any successfully completed STORE/UID STORE (with and without UNCHANGEDSINCE modifier), as well as to a FETCH command that implicitly sets \Seen flag: Adding the flag when it is already present or removing when it is not present SHOULD NOT change the mod-sequence. This will prevent spurious client synchronization requests. Melnikov & Hole Standards Track [Page 18] RFC 4551 IMAP Extension for Conditional STORE June 2006 However, note that client implementers MUST NOT rely on this server behavior. A client can't distinguish between the case when a server has violated the SHOULD mentioned above, and that when one or more clients set and unset (or unset and set) the flag in another session. 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) [ABNF] notation. Elements not defined here can be found in the formal syntax of the ABNF [ABNF], IMAP [IMAP4], and IMAP ABNF extensions [IMAPABNF] specifications. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper- or lowercase characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. capability =/ "CONDSTORE" status-att =/ "HIGHESTMODSEQ" ;; extends non-terminal defined in RFC 3501. status-att-val =/ "HIGHESTMODSEQ" SP mod-sequence-valzer ;; extends non-terminal defined in [IMAPABNF]. ;; Value 0 denotes that the mailbox doesn't ;; support persistent mod-sequences ;; as described in Section 3.1.2 store-modifier =/ "UNCHANGEDSINCE" SP mod-sequence-valzer ;; Only a single "UNCHANGEDSINCE" may be ;; specified in a STORE operation fetch-modifier =/ chgsince-fetch-mod ;; conforms to the generic "fetch-modifier" ;; syntax defined in [IMAPABNF]. chgsince-fetch-mod = "CHANGEDSINCE" SP mod-sequence-value ;; CHANGEDSINCE FETCH modifier conforms to ;; the fetch-modifier syntax fetch-att =/ fetch-mod-sequence ;; modifies original IMAP4 fetch-att fetch-mod-sequence = "MODSEQ" fetch-mod-resp = "MODSEQ" SP "(" permsg-modsequence ")" msg-att-dynamic =/ fetch-mod-resp Melnikov & Hole Standards Track [Page 19] RFC 4551 IMAP Extension for Conditional STORE June 2006 search-key =/ search-modsequence ;; modifies original IMAP4 search-key ;; ;; This change applies to all commands ;; referencing this non-terminal, in ;; particular SEARCH. search-modsequence = "MODSEQ" [search-modseq-ext] SP mod-sequence-valzer search-modseq-ext = SP entry-name SP entry-type-req resp-text-code =/ "HIGHESTMODSEQ" SP mod-sequence-value / "NOMODSEQ" / "MODIFIED" SP set entry-name = entry-flag-name entry-flag-name = DQUOTE "/flags/" attr-flag DQUOTE ;; each system or user defined flag ;; is mapped to "/flags/". ;; ;; follows the escape rules ;; used by "quoted" string as described in ;; Section 4.3 of [IMAP4], e.g., for the flag ;; \Seen the corresponding is ;; "/flags/\\seen", and for the flag ;; $MDNSent, the corresponding ;; is "/flags/$mdnsent". entry-type-resp = "priv" / "shared" ;; metadata item type entry-type-req = entry-type-resp / "all" ;; perform SEARCH operation on private ;; metadata item, shared metadata item or both permsg-modsequence = mod-sequence-value ;; per message mod-sequence mod-sequence-value = 1*DIGIT ;; Positive unsigned 64-bit integer ;; (mod-sequence) ;; (1 <= n < 18,446,744,073,709,551,615) mod-sequence-valzer = "0" / mod-sequence-value search-sort-mod-seq = "(" "MODSEQ" SP mod-sequence-value ")" Melnikov & Hole Standards Track [Page 20] RFC 4551 IMAP Extension for Conditional STORE June 2006 select-param =/ condstore-param ;; conforms to the generic "select-param" ;; non-terminal syntax defined in [IMAPABNF]. condstore-param = "CONDSTORE" mailbox-data =/ "SEARCH" [1*(SP nz-number) SP search-sort-mod-seq] attr-flag = "\\Answered" / "\\Flagged" / "\\Deleted" / "\\Seen" / "\\Draft" / attr-flag-keyword / attr-flag-extension ;; Does not include "\\Recent" attr-flag-extension = "\\" atom ;; Future expansion. Client implementations ;; MUST accept flag-extension flags. Server ;; implementations MUST NOT generate ;; flag-extension flags except as defined by ;; future standard or standards-track ;; revisions of [IMAP4]. attr-flag-keyword = atom 5. Server Implementation Considerations This section describes how a server implementation that doesn't store separate per-metadata mod-sequences for different metadata items can avoid sending the MODIFIED response to any of the following conditional STORE operations: +FLAGS -FLAGS +FLAGS.SILENT -FLAGS.SILENT Note that the optimization described in this section can't be performed in case of a conditional STORE FLAGS operation. Let's use the following example. The client has issued C: a106 STORE 100:150 (UNCHANGEDSINCE 212030000000) +FLAGS.SILENT ($Processed) When the server receives the command and parses it successfully, it iterates through the message set and tries to execute the conditional STORE command for each message. Melnikov & Hole Standards Track [Page 21] RFC 4551 IMAP Extension for Conditional STORE June 2006 Each server internally works as a client, i.e., it has to cache the current state of all IMAP flags as it is known to the client. In order to report flag changes to the client, the server compares the cached values with the values in its database for IMAP flags. Imagine that another client has changed the state of a flag \Deleted on the message 101 and that the change updated the mod-sequence for the message. The server knows that the mod-sequence for the mailbox has changed; however, it also knows that: a) the client is not interested in \Deleted flag, as it hasn't included it in +FLAGS.SILENT operation; and b) the state of the flag $Processed hasn't changed (the server can determine this by comparing cached flag state with the state of the flag in the database). Therefore, the server doesn't have to report MODIFIED to the client. Instead, the server may set $Processed flag, update the mod-sequence for the message 101 once again and send an untagged FETCH response with new mod-sequence and flags: S: * 101 FETCH (MODSEQ (303011130956) FLAGS ($Processed \Deleted \Answered)) See also Section 3.8 for additional quality-of-implementation issues. 6. Security Considerations It is believed that the Conditional STORE extension doesn't raise any new security concerns that are not already discussed in [IMAP4]. However, the availability of this extension may make it possible for IMAP4 to be used in critical applications it could not be used for previously, making correct IMAP server implementation and operation even more important. 7. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/imap4-capabilities This document defines the CONDSTORE IMAP capability. IANA has added it to the registry accordingly. Melnikov & Hole Standards Track [Page 22] RFC 4551 IMAP Extension for Conditional STORE June 2006 8. References 8.1. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. 8.2. Informative References [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [ANN] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work in Progress, March 2006. [NTP] Mills, D., "Network Time Protocol (Version 3) Specification, Implementation and Analysis", RFC 1305, March 1992. [RFC-2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC 2180, July 1997. 9. Acknowledgements Some text was borrowed from "IMAP ANNOTATE Extension" [ANN] by Randall Gellens and Cyrus Daboo and from "ACAP -- Application Configuration Access Protocol" [ACAP] by Chris Newman and John Myers. Many thanks to Randall Gellens for his thorough review of the document. The authors also acknowledge the feedback provided by Cyrus Daboo, Larry Greenfield, Chris Newman, Harrie Hazewinkel, Arnt Gulbrandsen, Timo Sirainen, Mark Crispin, Ned Freed, Ken Murchison, and Dave Cridland. Melnikov & Hole Standards Track [Page 23] RFC 4551 IMAP Extension for Conditional STORE June 2006 Authors' Addresses Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX, United Kingdom EMail: Alexey.Melnikov@isode.com Steve Hole ACI WorldWide/MessagingDirect #1807, 10088 102 Ave Edmonton, AB T5J 2Z1 Canada EMail: Steve.Hole@messagingdirect.com Melnikov & Hole Standards Track [Page 24] RFC 4551 IMAP Extension for Conditional STORE June 2006 Full Copyright Statement Copyright (C) The Internet Society (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is provided by the IETF Administrative Support Activity (IASA). Melnikov & Hole Standards Track [Page 25] ================================================ FILE: docs/rfcs/rfc4731.IMAP4_Extension_to_SEARCH_command.txt ================================================ Network Working Group A. Melnikov Request for Comments: 4731 Isode Ltd Category: Standards Track D. Cridland Inventure Systems Ltd November 2006 IMAP4 Extension to SEARCH Command for Controlling What Kind of Information Is Returned Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (C) The IETF Trust (2006). Abstract This document extends IMAP (RFC 3501) SEARCH and UID SEARCH commands with several result options, which can control what kind of information is returned. The following result options are defined: minimal value, maximal value, all found messages, and number of found messages. Table of Contents 1. Introduction ....................................................2 2. Conventions Used in This Document ...............................2 3. IMAP Protocol Changes ...........................................2 3.1. New SEARCH/UID SEARCH Result Options .......................2 3.2. Interaction with CONDSTORE extension .......................4 4. Formal Syntax ...................................................5 5. Security Considerations .........................................6 6. IANA Considerations .............................................6 7. Normative References ............................................6 8. Acknowledgments .................................................6 Melnikov & Cridland Standards Track [Page 1] RFC 4731 IMAP4 Extension to SEARCH November 2006 1. Introduction [IMAPABNF] extended SEARCH and UID SEARCH commands with result specifiers (also known as result options), which can control what kind of information is returned. A server advertising the ESEARCH capability supports the following result options: minimal value, maximal value, all found messages, and number of found messages. These result options allow clients to get SEARCH results in more convenient forms, while also saving bandwidth required to transport the results, for example, by finding the first unseen message or returning the number of unseen or deleted messages. Also, when a single MIN or a single MAX result option is specified, servers can optimize execution of SEARCHes. 2. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [KEYWORDS]. 3. IMAP Protocol Changes 3.1. New SEARCH/UID SEARCH Result Options The SEARCH/UID SEARCH commands are extended to allow for the following result options: MIN Return the lowest message number/UID that satisfies the SEARCH criteria. If the SEARCH results in no matches, the server MUST NOT include the MIN result option in the ESEARCH response; however, it still MUST send the ESEARCH response. MAX Return the highest message number/UID that satisfies the SEARCH criteria. If the SEARCH results in no matches, the server MUST NOT include the MAX result option in the ESEARCH response; however, it still MUST send the ESEARCH response. Melnikov & Cridland Standards Track [Page 2] RFC 4731 IMAP4 Extension to SEARCH November 2006 ALL Return all message numbers/UIDs that satisfy the SEARCH criteria. Unlike regular (unextended) SEARCH, the messages are always returned using the sequence-set syntax. A sequence-set representation may be more compact and can be used as is in a subsequent command that accepts sequence-set. Note, the client MUST NOT assume that messages/UIDs will be listed in any particular order. If the SEARCH results in no matches, the server MUST NOT include the ALL result option in the ESEARCH response; however, it still MUST send the ESEARCH response. COUNT Return number of the messages that satisfy the SEARCH criteria. This result option MUST always be included in the ESEARCH response. If one or more result options described above are specified, the extended SEARCH command MUST return a single ESEARCH response [IMAPABNF], instead of the SEARCH response. An extended UID SEARCH command MUST cause an ESEARCH response with the UID indicator present. Note that future extensions to this document can allow servers to return multiple ESEARCH responses for a single extended SEARCH command. These extensions will have to describe how results from multiple ESEARCH responses are to be amalgamated. If the list of result options is empty, that requests the server to return an ESEARCH response instead of the SEARCH response. This is equivalent to "(ALL)". Example: C: A282 SEARCH RETURN (MIN COUNT) FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" S: * ESEARCH (TAG "A282") MIN 2 COUNT 3 S: A282 OK SEARCH completed Example: C: A283 SEARCH RETURN () FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" S: * ESEARCH (TAG "A283") ALL 2,10:11 S: A283 OK SEARCH completed The following example demonstrates finding the first unseen message as returned in the UNSEEN response code on a successful SELECT command: Melnikov & Cridland Standards Track [Page 3] RFC 4731 IMAP4 Extension to SEARCH November 2006 Example: C: A284 SEARCH RETURN (MIN) UNSEEN S: * ESEARCH (TAG "A284") MIN 4 S: A284 OK SEARCH completed The following example demonstrates that if the ESEARCH UID indicator is present, all data in the ESEARCH response is referring to UIDs; for example, the MIN result specifier will be followed by a UID. Example: C: A285 UID SEARCH RETURN (MIN MAX) 1:5000 S: * ESEARCH (TAG "A285") UID MIN 7 MAX 3800 S: A285 OK SEARCH completed The following example demonstrates returning the number of deleted messages: Example: C: A286 SEARCH RETURN (COUNT) DELETED S: * ESEARCH (TAG "A286") COUNT 15 S: A286 OK SEARCH completed 3.2. Interaction with CONDSTORE extension When the server supports both the ESEARCH and the CONDSTORE [CONDSTORE] extension, and the client requests one or more result option described in section 3.1 together with the MODSEQ search criterion in the same SEARCH/UID SEARCH command, then the server MUST return the ESEARCH response containing the MODSEQ result option (described in the following paragraph) instead of the extended SEARCH response described in section 3.5 of [CONDSTORE]. If the SEARCH/UID SEARCH command contained a single MIN or MAX result option, the MODSEQ result option contains the mod-sequence for the found message. If the SEARCH/UID SEARCH command contained both MIN and MAX result options and no ALL/COUNT option, the MODSEQ result option contains the highest mod-sequence for the two returned messages. Otherwise the MODSEQ result option contains the highest mod-sequence for all messages being returned. Example: The following example demonstrates how Example 15 from [CONDSTORE] would look in the presence of one or more result option: C: a1 SEARCH RETURN (MIN) MODSEQ "/flags/\\draft" all 620162338 S: * ESEARCH (TAG "a1") MIN 2 MODSEQ 917162488 S: a1 OK Search complete C: a2 SEARCH RETURN (MAX) MODSEQ "/flags/\\draft" all 620162338 S: * ESEARCH (TAG "a2") MAX 23 MODSEQ 907162321 Melnikov & Cridland Standards Track [Page 4] RFC 4731 IMAP4 Extension to SEARCH November 2006 S: a2 OK Search complete C: a3 SEARCH RETURN (MIN MAX) MODSEQ "/flags/\\draft" all 620162338 S: * ESEARCH (TAG "a3") MIN 2 MAX 23 MODSEQ 917162488 S: a3 OK Search complete C: a4 SEARCH RETURN (MIN COUNT) MODSEQ "/flags/\\draft" all 620162338 S: * ESEARCH (TAG "a4") MIN 2 COUNT 10 MODSEQ 917162500 S: a4 OK Search complete 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. Non-terminals referenced but not defined below are as defined by [IMAP4], [CONDSTORE], or [IMAPABNF]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lowercase characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. capability =/ "ESEARCH" search-return-data = "MIN" SP nz-number / "MAX" SP nz-number / "ALL" SP sequence-set / "COUNT" SP number ;; conforms to the generic ;; search-return-data syntax defined ;; in [IMAPABNF] search-return-opt = "MIN" / "MAX" / "ALL" / "COUNT" ;; conforms to generic search-return-opt ;; syntax defined in [IMAPABNF] When the CONDSTORE [CONDSTORE] IMAP extension is also supported, the ABNF is updated as follows: search-return-data =/ "MODSEQ" SP mod-sequence-value ;; mod-sequence-value is defined ;; in [CONDSTORE] Melnikov & Cridland Standards Track [Page 5] RFC 4731 IMAP4 Extension to SEARCH November 2006 5. Security Considerations In the general case, the IMAP SEARCH/UID SEARCH commands can be CPU and/or IO intensive, and are seen by some as a potential attack point for denial of service attacks, so some sites/implementations even disable them entirely. This is quite unfortunate, as SEARCH command is one of the best examples demonstrating IMAP advantage over POP3. The ALL and COUNT return options don't change how SEARCH is working internally; they only change how information about found messages is returned. MIN and MAX SEARCH result options described in this document can lighten the load on IMAP servers that choose to optimize SEARCHes containing only one or both of them. It is believed that this extension doesn't raise any additional security concerns not already discussed in [IMAP4]. 6. IANA Considerations IMAP4 capabilities are registered by publishing a standards track RFC or an IESG-approved experimental RFC. The registry is currently located at . This document defines the ESEARCH IMAP capability, which IANA added to the registry. 7. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [ABNF] Crocker, D. (Ed.) and P. Overell , "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006.. [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE", RFC 4551, June 2006. 8. Acknowledgments Thanks to Michael Wener, Arnt Gulbrandsen, Cyrus Daboo, Mark Crispin, and Pete Maclean for comments and corrections. Melnikov & Cridland Standards Track [Page 6] RFC 4731 IMAP4 Extension to SEARCH November 2006 Authors' Addresses Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex, TW12 2BX UK EMail: Alexey.Melnikov@isode.com Dave A. Cridland Inventure Systems Limited EMail: dave.cridland@inventuresystems.co.uk URL: http://invsys.co.uk/dave/ Melnikov & Cridland Standards Track [Page 7] RFC 4731 IMAP4 Extension to SEARCH November 2006 Full Copyright Statement Copyright (C) The IETF Trust (2006). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST, AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Acknowledgement Funding for the RFC Editor function is currently provided by the Internet Society. Melnikov & Cridland Standards Track [Page 8] ================================================ FILE: docs/rfcs/rfc4978.IMAP_Compress_extension.txt ================================================ Network Working Group A. Gulbrandsen Request for Comments: 4978 Oryx Mail Systems GmbH Category: Standards Track August 2007 The IMAP COMPRESS Extension Status of this Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract The COMPRESS extension allows an IMAP connection to be effectively and efficiently compressed. Table of Contents 1. Introduction and Overview .......................................2 2. Conventions Used in This Document ...............................2 3. The COMPRESS Command ............................................3 4. Compression Efficiency ..........................................4 5. Formal Syntax ...................................................6 6. Security Considerations .........................................6 7. IANA Considerations .............................................6 8. Acknowledgements ................................................7 9. References ......................................................7 9.1. Normative References .......................................7 9.2. Informative References .....................................7 Gulbrandsen Standards Track [Page 1] RFC 4978 The IMAP COMPRESS Extension August 2007 1. Introduction and Overview A server which supports the COMPRESS extension indicates this with one or more capability names consisting of "COMPRESS=" followed by a supported compression algorithm name as described in this document. The goal of COMPRESS is to reduce the bandwidth usage of IMAP. Compared to PPP compression (see [RFC1962]) and modem-based compression (see [MNP] and [V42BIS]), COMPRESS offers much better compression efficiency. COMPRESS can be used together with Transport Security Layer (TLS) [RFC4346], Simple Authentication and Security layer (SASL) encryption, Virtual Private Networks (VPNs), etc. Compared to TLS compression [RFC3749], COMPRESS has the following (dis)advantages: - COMPRESS can be implemented easily both by IMAP servers and clients. - IMAP COMPRESS benefits from an intimate knowledge of the IMAP protocol's state machine, allowing for dynamic and aggressive optimization of the underlying compression algorithm's parameters. - When the TLS layer implements compression, any protocol using that layer can transparently benefit from that compression (e.g., SMTP and IMAP). COMPRESS is specific to IMAP. In order to increase interoperation, it is desirable to have as few different compression algorithms as possible, so this document specifies only one. The DEFLATE algorithm (defined in [RFC1951]) is standard, widely available and fairly efficient, so it is the only algorithm defined by this document. In order to increase interoperation, IMAP servers that advertise this extension SHOULD also advertise the TLS DEFLATE compression mechanism as defined in [RFC3749]. IMAP clients MAY use either COMPRESS or TLS compression, however, if the client and server support both, it is RECOMMENDED that the client choose TLS compression. The extension adds one new command (COMPRESS) and no new responses. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. Formal syntax is defined by [RFC4234] as modified by [RFC3501]. Gulbrandsen Standards Track [Page 2] RFC 4978 The IMAP COMPRESS Extension August 2007 In the examples, "C:" and "S:" indicate lines sent by the client and server respectively. "[...]" denotes elision. 3. The COMPRESS Command Arguments: Name of compression mechanism: "DEFLATE". Responses: None Result: OK The server will compress its responses and expects the client to compress its commands. NO Compression is already active via another layer. BAD Command unknown, invalid or unknown argument, or COMPRESS already active. The COMPRESS command instructs the server to use the named compression mechanism ("DEFLATE" is the only one defined) for all commands and/or responses after COMPRESS. The client MUST NOT send any further commands until it has seen the result of COMPRESS. If the response was OK, the client MUST compress starting with the first command after COMPRESS. If the server response was BAD or NO, the client MUST NOT turn on compression. If the server responds NO because it knows that the same mechanism is active already (e.g., because TLS has negotiated the same mechanism), it MUST send COMPRESSIONACTIVE as resp-text-code (see [RFC3501], Section 7.1), and the resp-text SHOULD say which layer compresses. If the server issues an OK response, the server MUST compress starting immediately after the CRLF which ends the tagged OK response. (Responses issued by the server before the OK response will, of course, still be uncompressed.) If the server issues a BAD or NO response, the server MUST NOT turn on compression. For DEFLATE (as for many other compression mechanisms), the compressor can trade speed against quality. When decompressing there isn't much of a tradeoff. Consequently, the client and server are both free to pick the best reasonable rate of compression for the data they send. When COMPRESS is combined with TLS (see [RFC4346]) or SASL (see [RFC4422]) security layers, the sending order of the three extensions MUST be first COMPRESS, then SASL, and finally TLS. That is, before data is transmitted it is first compressed. Second, if a SASL security layer has been negotiated, the compressed data is then signed and/or encrypted accordingly. Third, if a TLS security layer has been negotiated, the data from the previous step is signed and/or Gulbrandsen Standards Track [Page 3] RFC 4978 The IMAP COMPRESS Extension August 2007 encrypted accordingly. When receiving data, the processing order MUST be reversed. This ensures that before sending, data is compressed before it is encrypted, independent of the order in which the client issues COMPRESS, AUTHENTICATE, and STARTTLS. The following example illustrates how commands and responses are compressed during a simple login sequence: S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] C: a starttls S: a OK TLS active From this point on, everything is encrypted. C: b login arnt tnra S: b OK Logged in as arnt C: c compress deflate S: d OK DEFLATE active From this point on, everything is compressed before being encrypted. The following example demonstrates how a server may refuse to compress twice: S: * OK [CAPABILITY IMAP4REV1 STARTTLS COMPRESS=DEFLATE] [...] C: c compress deflate S: c NO [COMPRESSIONACTIVE] DEFLATE active via TLS 4. Compression Efficiency This section is informative, not normative. IMAP poses some unusual problems for a compression layer. Upstream is fairly simple. Most IMAP clients send the same few commands again and again, so any compression algorithm that can exploit repetition works efficiently. The APPEND command is an exception; clients that send many APPEND commands may want to surround large literals with flushes in the same way as is recommended for servers later in this section. Downstream has the unusual property that several kinds of data are sent, confusing all dictionary-based compression algorithms. Gulbrandsen Standards Track [Page 4] RFC 4978 The IMAP COMPRESS Extension August 2007 One type is IMAP responses. These are highly compressible; zlib using its least CPU-intensive setting compresses typical responses to 25-40% of their original size. Another type is email headers. These are equally compressible, and benefit from using the same dictionary as the IMAP responses. A third type is email body text. Text is usually fairly short and includes much ASCII, so the same compression dictionary will do a good job here, too. When multiple messages in the same thread are read at the same time, quoted lines etc. can often be compressed almost to zero. Finally, attachments (non-text email bodies) are transmitted, either in binary form or encoded with base-64. When attachments are retrieved in binary form, DEFLATE may be able to compress them, but the format of the attachment is usually not IMAP- like, so the dictionary built while compressing IMAP does not help. The compressor has to adapt its dictionary from IMAP to the attachment's format, and then back. A few file formats aren't compressible at all using deflate, e.g., .gz, .zip, and .jpg files. When attachments are retrieved in base-64 form, the same problems apply, but the base-64 encoding adds another problem. 8-bit compression algorithms such as deflate work well on 8-bit file formats, however base-64 turns a file into something resembling 6-bit bytes, hiding most of the 8-bit file format from the compressor. When using the zlib library (see [RFC1951]), the functions deflateInit2(), deflate(), inflateInit2(), and inflate() suffice to implement this extension. The windowBits value must be in the range -8 to -15, or else deflateInit2() uses the wrong format. deflateParams() can be used to improve compression rate and resource use. The Z_FULL_FLUSH argument to deflate() can be used to clear the dictionary (the receiving peer does not need to do anything). A client can improve downstream compression by implementing BINARY (defined in [RFC3516]) and using FETCH BINARY instead of FETCH BODY. In the author's experience, the improvement ranges from 5% to 40% depending on the attachment being downloaded. A server can improve downstream compression if it hints to the compressor that the data type is about to change strongly, e.g., by sending a Z_FULL_FLUSH at the start and end of large non-text literals (before and after '*CHAR8' in the definition of literal in RFC 3501, page 86). Small literals are best left alone. A possible boundary is 5k. Gulbrandsen Standards Track [Page 5] RFC 4978 The IMAP COMPRESS Extension August 2007 A server can improve the CPU efficiency both of the server and the client if it adjusts the compression level (e.g., using the deflateParams() function in zlib) at these points, to avoid trying to compress incompressible attachments. A very simple strategy is to change the level to 0 at the start of a literal provided the first two bytes are either 0x1F 0x8B (as in deflate-compressed files) or 0xFF 0xD8 (JPEG), and to keep it at 1-5 the rest of the time. More complex strategies are possible. 5. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC4234]. This syntax augments the grammar specified in [RFC3501]. [RFC4234] defines SP and [RFC3501] defines command-auth, capability, and resp-text-code. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. command-auth =/ compress compress = "COMPRESS" SP algorithm capability =/ "COMPRESS=" algorithm ;; multiple COMPRESS capabilities allowed algorithm = "DEFLATE" resp-text-code =/ "COMPRESSIONACTIVE" Note that due the syntax of capability names, future algorithm names must be atoms. 6. Security Considerations As for TLS compression [RFC3749]. 7. IANA Considerations The IANA has added COMPRESS=DEFLATE to the list of IMAP capabilities. Gulbrandsen Standards Track [Page 6] RFC 4978 The IMAP COMPRESS Extension August 2007 8. Acknowledgements Eric Burger, Dave Cridland, Tony Finch, Ned Freed, Philip Guenther, Randall Gellens, Tony Hansen, Cullen Jennings, Stephane Maes, Alexey Melnikov, Lyndon Nerenberg, and Zoltan Ordogh have all helped with this document. The author would also like to thank various people in the rooms at meetings, whose help is real, but not reflected in the author's mailbox. 9. References 9.1. Normative References [RFC1951] Deutsch, P., "DEFLATE Compressed Data Format Specification version 1.3", RFC 1951, May 1996. [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC4234] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. 9.2. Informative References [RFC1962] Rand, D., "The PPP Compression Control Protocol (CCP)", RFC 1962, June 1996. [RFC3516] Nerenberg, L., "IMAP4 Binary Content Extension", RFC 3516, April 2003. [RFC3749] Hollenbeck, S., "Transport Layer Security Protocol Compression Methods", RFC 3749, May 2004. [RFC4346] Dierks, T. and E. Rescorla, "The Transport Layer Security (TLS) Protocol Version 1.1", RFC 4346, April 2006. [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and Security Layer (SASL)", RFC 4422, June 2006. [V42BIS] ITU, "V.42bis: Data compression procedures for data circuit-terminating equipment (DCE) using error correction procedures", http://www.itu.int/rec/T-REC-V.42bis, January 1990. Gulbrandsen Standards Track [Page 7] RFC 4978 The IMAP COMPRESS Extension August 2007 [MNP] Gilbert Held, "The Complete Modem Reference", Second Edition, Wiley Professional Computing, ISBN 0-471-00852-4, May 1994. Author's Address Arnt Gulbrandsen Oryx Mail Systems GmbH Schweppermannstr. 8 D-81671 Muenchen Germany Fax: +49 89 4502 9758 EMail: arnt@oryx.com Gulbrandsen Standards Track [Page 8] RFC 4978 The IMAP COMPRESS Extension August 2007 Full Copyright Statement Copyright (C) The IETF Trust (2007). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Gulbrandsen Standards Track [Page 9] ================================================ FILE: docs/rfcs/rfc5032.IMAP_WITHIN_Search_extension.txt ================================================ Network Working Group E. Burger, Ed. Request for Comments: 5032 BEA Systems, Inc. Updates: 3501 September 2007 Category: Standards Track WITHIN Search Extension to the IMAP Protocol Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract This document describes the WITHIN extension to IMAP SEARCH. IMAP SEARCH returns messages whose internal date is within or outside a specified interval. The mechanism described here, OLDER and YOUNGER, differs from BEFORE and SINCE in that the client specifies an interval, rather than a date. WITHIN is useful for persistent searches where either the device does not have the capacity to perform the search at regular intervals or the network is of limited bandwidth and thus there is a desire to reduce network traffic from sending repeated requests and redundant responses. 1. Introduction This extension exposes two new search keys, OLDER and YOUNGER, each of which takes a non-zero integer argument corresponding to a time interval in seconds. The server calculates the time of interest by subtracting the time interval the client presents from the current date and time of the server. The server then either returns messages older or younger than the resultant time and date, depending on the search key used. 1.1. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [RFC2119]. Burger Standards Track [Page 1] RFC 5032 Search Within September 2007 When describing the general syntax, we omit some definitions, as RFC 3501 [RFC3501] defines them. 2. Protocol Operation An IMAP4 server that supports the capability described here MUST return "WITHIN" as one of the server supported capabilities in the CAPABILITY command. For both the OLDER and YOUNGER search keys, the server calculates a target date and time by subtracting the interval, specified in seconds, from the current date and time of the server. The server then compares the target time with the INTERNALDATE of the message, as specified in IMAP [RFC3501]. For OLDER, messages match if the INTERNALDATE is less recent than or equal to the target time. For YOUNGER, messages match if the INTERNALDATE is more recent than or equal to the target time. Both OLDER and YOUNGER searches always result in exact matching, to the resolution of a second. However, if one is doing a dynamic evaluation, for example, in a context [CONTEXT], one needs to be aware that the server might perform the evaluation periodically. Thus, the server may delay the updates. Clients MUST be aware that dynamic search results may not reflect the current state of the mailbox. If the client needs a search result that reflects the current state of the mailbox, we RECOMMEND that the client issue a new search. 3. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation. Elements not defined here can be found in the formal syntax of ABNF [RFC4234] and IMAP [RFC3501]. This document extends RFC 3501 [RFC3501] with two new search keys: OLDER and YOUNGER . search-key =/ ( "OLDER" / "YOUNGER" ) SP nz-number ; search-key defined in RFC 3501 4. Example C: a1 SEARCH UNSEEN YOUNGER 259200 S: a1 * SEARCH 4 8 15 16 23 42 Search for all unseen messages within the past 3 days, or 259200 seconds, according to the server's current time. Burger Standards Track [Page 2] RFC 5032 Search Within September 2007 5. Security Considerations The WITHIN extension does not raise any security considerations that are not present in the base protocol. Considerations are the same as for IMAP [RFC3501]. 6. IANA Considerations Per the IMAP RFC [RFC3501], registration of a new IMAP capability in the IMAP Capability registry requires the publication of a standards- track RFC or an IESG approved experimental RFC. The registry is currently located at . This standards-track document defines the WITHIN IMAP capability. IANA has added this extension to the IANA IMAP Capability registry. 7. References 7.1. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, BCP 14, March 1997. [RFC3501] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [RFC4234] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. 7.2. Informative References [CONTEXT] Melnikov, D. and C. King, "Contexts for IMAP4", Work in Progress, May 2006. Burger Standards Track [Page 3] RFC 5032 Search Within September 2007 Appendix A. Contributors Stephane Maes and Ray Cromwell wrote the original version of this document as part of P-IMAP, as well as the first versions for the IETF. From an attribution perspective, they are clearly authors. Appendix B. Acknowledgements The authors want to thank all who have contributed key insight and who have extensively reviewed and discussed the concepts of LPSEARCH. They also thank the authors of its early introduction in P-IMAP. We also want to give a special thanks to Arnt Gilbrandsen, Ken Murchison, Zoltan Ordogh, and most especially Dave Cridland for their review and suggestions. A special thank you goes to Alexey Melnikov for his choice submission of text. Author's Address Eric W. Burger (editor) BEA Systems, Inc. USA EMail: eric.burger@bea.com URI: http://www.standardstrack.com Burger Standards Track [Page 4] RFC 5032 Search Within September 2007 Full Copyright Statement Copyright (C) The IETF Trust (2007). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Burger Standards Track [Page 5] ================================================ FILE: docs/rfcs/rfc5161.IMAP_ENABLE_extension.txt ================================================ Network Working Group A. Gulbrandsen, Ed. Request for Comments: 5161 Oryx Mail Systems GmbH Category: Standards Track A. Melnikov, Ed. Isode Limited March 2008 The IMAP ENABLE Extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract Most IMAP extensions are used by the client when it wants to and the server supports it. However, a few extensions require the server to know whether a client supports that extension. The ENABLE extension allows an IMAP client to say which extensions it supports. 1. Overview Several IMAP extensions allow the server to return unsolicited responses specific to these extensions in certain circumstances. However, servers cannot send those unsolicited responses until they know that the clients support such extensions and thus won't choke on the extension response data. Up until now, extensions have typically stated that a server cannot send the unsolicited responses until after the client has used a command with the extension data (i.e., at that point the server knows the client is aware of the extension). CONDSTORE ([RFC4551]), ANNOTATE ([ANNOTATE]), and some extensions under consideration at the moment use various commands to enable server extensions. For example, CONDSTORE uses a SELECT or FETCH parameter, and ANNOTATE uses a side effect of FETCH. The ENABLE extension provides an explicit indication from the client that it supports particular extensions. This is done using a new ENABLE command. An IMAP server that supports ENABLE advertises this by including the word ENABLE in its capability list. Gulbrandsen & Melnikov Standards Track [Page 1] RFC 5161 The IMAP ENABLE Extension March 2008 Most IMAP extensions do not require the client to enable the extension in any way. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. Formal syntax is defined by [RFC5234] and [RFC3501]. Example lines prefaced by "C:" are sent by the client and ones prefaced by "S:" by the server. The five characters [...] means that something has been elided. 3. Protocol Changes 3.1. The ENABLE Command Arguments: capability names Result: OK: Relevant capabilities enabled BAD: No arguments, or syntax error in an argument The ENABLE command takes a list of capability names, and requests the server to enable the named extensions. Once enabled using ENABLE, each extension remains active until the IMAP connection is closed. For each argument, the server does the following: - If the argument is not an extension known to the server, the server MUST ignore the argument. - If the argument is an extension known to the server, and it is not specifically permitted to be enabled using ENABLE, the server MUST ignore the argument. (Note that knowing about an extension doesn't necessarily imply supporting that extension.) - If the argument is an extension that is supported by the server and that needs to be enabled, the server MUST enable the extension for the duration of the connection. At present, this applies only to CONDSTORE ([RFC4551]). Note that once an extension is enabled, there is no way to disable it. If the ENABLE command is successful, the server MUST send an untagged ENABLED response (see Section 3.2). Gulbrandsen & Melnikov Standards Track [Page 2] RFC 5161 The IMAP ENABLE Extension March 2008 Clients SHOULD only include extensions that need to be enabled by the server. At the time of publication, CONDSTORE is the only such extension (i.e., ENABLE CONDSTORE is an additional "CONDSTORE enabling command" as defined in [RFC4551]). Future RFCs may add to this list. The ENABLE command is only valid in the authenticated state (see [RFC3501]), before any mailbox is selected. Clients MUST NOT issue ENABLE once they SELECT/EXAMINE a mailbox; however, server implementations don't have to check that no mailbox is selected or was previously selected during the duration of a connection. The ENABLE command can be issued multiple times in a session. It is additive; i.e., "ENABLE a b", followed by "ENABLE c" is the same as a single command "ENABLE a b c". When multiple ENABLE commands are issued, each corresponding ENABLED response SHOULD only contain extensions enabled by the corresponding ENABLE command. There are no limitations on pipelining ENABLE. For example, it is possible to send ENABLE and then immediately SELECT, or a LOGIN immediately followed by ENABLE. The server MUST NOT change the CAPABILITY list as a result of executing ENABLE; i.e., a CAPABILITY command issued right after an ENABLE command MUST list the same capabilities as a CAPABILITY command issued before the ENABLE command. This is demonstrated in the following example: C: t1 CAPABILITY S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA S: t1 OK foo C: t2 ENABLE CONDSTORE X-GOOD-IDEA S: * ENABLED X-GOOD-IDEA S: t2 OK foo C: t3 CAPABILITY S: * CAPABILITY IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA S: t3 OK foo again In the following example, the client enables CONDSTORE: C: a1 ENABLE CONDSTORE S: * ENABLED CONDSTORE S: a1 OK Conditional Store enabled Gulbrandsen & Melnikov Standards Track [Page 3] RFC 5161 The IMAP ENABLE Extension March 2008 3.2. The ENABLED Response Contents: capability listing The ENABLED response occurs as a result of an ENABLE command. The capability listing contains a space-separated listing of capability names that the server supports and that were successfully enabled. The ENABLED response may contain no capabilities, which means that no extensions listed by the client were successfully enabled. 3.3. Note to Designers of Extensions That May Use the ENABLE Command Designers of IMAP extensions are discouraged from creating extensions that require ENABLE unless there is no good alternative design. Specifically, extensions that cause potentially incompatible behavior changes to deployed server responses (and thus benefit from ENABLE) have a higher complexity cost than extensions that do not. 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234] including the core rules in Appendix B.1. [RFC3501] defines the non-terminals "capability" and "command-any". Except as noted otherwise, all alphabetic characters are case-insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. capability =/ "ENABLE" command-any =/ "ENABLE" 1*(SP capability) response-data =/ "*" SP enable-data CRLF enable-data = "ENABLED" *(SP capability) 5. Security Considerations It is believed that this extension doesn't add any security considerations that are not already present in the base IMAP protocol [RFC3501]. 6. IANA Considerations The IANA has added ENABLE to the IMAP4 Capabilities Registry. Gulbrandsen & Melnikov Standards Track [Page 4] RFC 5161 The IMAP ENABLE Extension March 2008 7. Acknowledgments The editors would like to thank Randy Gellens, Chris Newman, Peter Coates, Dave Cridland, Mark Crispin, Ned Freed, Dan Karp, Cyrus Daboo, Ken Murchison, and Eric Burger for comments and corrections. However, this doesn't necessarily mean that they endorse this extension, agree with all details, or are responsible for errors introduced by the editors. 8. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. 9. Informative References [ANNOTATE] Daboo, C. and R. Gellens, "IMAP ANNOTATE Extension", Work in Progress, August 2006. Gulbrandsen & Melnikov Standards Track [Page 5] RFC 5161 The IMAP ENABLE Extension March 2008 Editors' Addresses Arnt Gulbrandsen Oryx Mail Systems GmbH Schweppermannstr. 8 D-81671 Muenchen Germany Fax: +49 89 4502 9758 EMail: arnt@oryx.com Alexey Melnikov Isode Ltd 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com Gulbrandsen & Melnikov Standards Track [Page 6] RFC 5161 The IMAP ENABLE Extension March 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Gulbrandsen & Melnikov Standards Track [Page 7] ================================================ FILE: docs/rfcs/rfc5162.IMAP4_Extensions_for_Quick_Mailbox_resync.txt ================================================ Network Working Group A. Melnikov Request for Comments: 5162 D. Cridland Category: Standards Track Isode Ltd C. Wilson Nokia March 2008 IMAP4 Extensions for Quick Mailbox Resynchronization Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract This document defines an IMAP4 extension, which gives an IMAP client the ability to quickly resynchronize any previously opened mailbox as part of the SELECT command, without the need for server-side state or additional client round-trips. This extension also introduces a new response that allows for a more compact representation of a list of expunged messages (and always includes the Unique Identifiers (UIDs) expunged). Melnikov, et al. Standards Track [Page 1] RFC 5162 IMAP Quick Mailbox Resync March 2008 Table of Contents 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 2. Requirements Notation . . . . . . . . . . . . . . . . . . . . 4 3. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 4 3.1. QRESYNC Parameter to SELECT/EXAMINE . . . . . . . . . . . 4 3.2. VANISHED UID FETCH Modifier . . . . . . . . . . . . . . . 8 3.3. EXPUNGE Command . . . . . . . . . . . . . . . . . . . . . 10 3.4. CLOSE Command . . . . . . . . . . . . . . . . . . . . . . 11 3.5. UID EXPUNGE Command . . . . . . . . . . . . . . . . . . . 11 3.6. VANISHED Response . . . . . . . . . . . . . . . . . . . . 12 3.7. CLOSED Response Code . . . . . . . . . . . . . . . . . . . 15 4. Server Implementation Considerations . . . . . . . . . . . . . 15 4.1. Server Implementations That Don't Store Extra State . . . 15 4.2. Server Implementations Storing Minimal State . . . . . . . 16 4.3. Additional State Required on the Server . . . . . . . . . 16 5. Updated Synchronization Sequence . . . . . . . . . . . . . . . 17 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 7. Security Considerations . . . . . . . . . . . . . . . . . . . 20 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 21 9. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 21 10. References . . . . . . . . . . . . . . . . . . . . . . . . . . 21 10.1. Normative References . . . . . . . . . . . . . . . . . . . 21 10.2. Informative References . . . . . . . . . . . . . . . . . . 22 1. Introduction and Overview The [CONDSTORE] extension gives a disconnected client the ability to quickly resynchronize IMAP flag changes for previously seen messages. This can be done using the CHANGEDSINCE FETCH modifier once a mailbox is opened. In order for the client to discover which messages have been expunged, the client still has to issue a UID FETCH or a UID SEARCH command. This document defines an extension to [CONDSTORE] that allows a reconnecting client to perform full resynchronization, including discovery of expunged messages, in a single round-trip. This extension also introduces a new response, VANISHED, that allows for a more compact representation of a list of expunged messages. This extension can be useful for mobile clients that can experience frequent disconnects caused by environmental factors (battery life, signal strength, etc.). Such clients need a way to quickly reconnect to the IMAP server, while minimizing delay experienced by the user as well as the amount of traffic (and hence the expense) generated by resynchronization. Melnikov, et al. Standards Track [Page 2] RFC 5162 IMAP Quick Mailbox Resync March 2008 By extending the SELECT command to perform the additional resynchronization, this also allows clients to reduce concurrent connections to the IMAP server held purely for the sake of avoiding the resynchronization. The quick resync IMAP extension is present if an IMAP4 server returns "QRESYNC" as one of the supported capabilities to the CAPABILITY command. Servers supporting this extension MUST implement and advertise support for the [ENABLE] IMAP extension. Also, the presence of the "QRESYNC" capability implies support for the [CONDSTORE] IMAP extension even if the CONDSTORE capability isn't advertised. A server compliant with this specification is REQUIREd to support "ENABLE QRESYNC" and "ENABLE QRESYNC CONDSTORE" (which are "CONDSTORE enabling commands", as defined in [CONDSTORE], and have identical results), but there is no requirement for a compliant server to support "ENABLE CONDSTORE" by itself. The "ENABLE QRESYNC"/"ENABLE QRESYNC CONDSTORE" command also tells the server that it SHOULD start sending VANISHED responses (see Section 3.6) instead of EXPUNGE responses. This change remains in effect until the connection is closed. For compatibility with clients that only support the [CONDSTORE] IMAP extension, servers SHOULD advertise CONDSTORE in the CAPABILITY response as well. A client making use of this extension MUST issue "ENABLE QRESYNC" once it is authenticated. A server MUST respond with a tagged BAD response if the QRESYNC parameter to the SELECT/EXAMINE command or the VANISHED UID FETCH modifier is specified and the client hasn't issued "ENABLE QRESYNC" in the current connection. This document puts additional requirements on a server implementing the [CONDSTORE] extension. Each mailbox that supports persistent storage of mod-sequences, i.e., for which the server has sent a HIGHESTMODSEQ untagged OK response code on a successful SELECT/ EXAMINE, MUST increment the per-mailbox mod-sequence when one or more messages are expunged due to EXPUNGE, UID EXPUNGE or CLOSE; the server MUST associate the incremented mod-sequence with the UIDs of the expunged messages. A client that supports CONDSTORE but not this extension might resynchronize a mailbox and discover that its HIGHESTMODSEQ has increased from the value cached by the client. If the increase is only due to messages having been expunged since the client last synchronized, the client is likely to send a FETCH ... CHANGEDSINCE command that returns no data. Thus, a client that supports CONDSTORE Melnikov, et al. Standards Track [Page 3] RFC 5162 IMAP Quick Mailbox Resync March 2008 but not this extension might incur a penalty of an unneeded round- trip when resynchronizing some mailboxes (those that have had messages expunged but no flag changes since the last synchronization). This extra round-trip is only incurred by clients that support CONDSTORE but not this extension, and only when a mailbox has had messages expunged but no flag changes to non-expunged messages. Since CONDSTORE is a relatively new extension, it is thought likely that clients that support it will also support this extension. 2. Requirements Notation The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. In examples, "C:" and "S:" indicate lines sent by the client and server respectively. If a single "C:" or "S:" label applies to multiple lines, then the line breaks between those lines are for editorial clarity only and are not part of the actual protocol exchange. The five characters [...] means that something has been elided. Understanding of the IMAP message sequence numbers and UIDs and the EXPUNGE response [RFC3501] is essential when reading this document. 3. IMAP Protocol Changes 3.1. QRESYNC Parameter to SELECT/EXAMINE The Quick Resynchronization parameter to SELECT/EXAMINE commands has four arguments: o the last known UIDVALIDITY, o the last known modification sequence, o the optional set of known UIDs, and o an optional parenthesized list of known sequence ranges and their corresponding UIDs. A server MUST respond with a tagged BAD response if the Quick Resynchronization parameter to SELECT/EXAMINE command is specified and the client hasn't issued "ENABLE QRESYNC" in the current connection. Melnikov, et al. Standards Track [Page 4] RFC 5162 IMAP Quick Mailbox Resync March 2008 Before opening the specified mailbox, the server verifies all arguments for syntactic validity. If any parameter is not syntactically valid, the server returns the tagged BAD response, and the mailbox remains unselected. Once the check is done, the server opens the mailbox as if no SELECT/EXAMINE parameters are specified (this is subject to processing of other parameters as defined in other extensions). In particular this means that the server MUST send all untagged responses as specified in Sections 6.3.1 and 6.3.2 of [RFC3501]. After that, the server checks the UIDVALIDITY value provided by the client. If the provided UIDVALIDITY doesn't match the UIDVALIDITY for the mailbox being opened, then the server MUST ignore the remaining parameters and behave as if no dynamic message data changed. The client can discover this situation by comparing the UIDVALIDITY value returned by the server. This behavior allows the client not to synchronize the mailbox or decide on the best synchronization strategy. Example: Attempting to resynchronize INBOX, but the provided UIDVALIDITY parameter doesn't match the current UIDVALIDITY value. C: A02 SELECT INBOX (QRESYNC (67890007 20050715194045000 41,43:211,214:541)) S: * 464 EXISTS S: * 3 RECENT S: * OK [UIDVALIDITY 3857529045] UIDVALIDITY S: * OK [UIDNEXT 550] Predicted next UID S: * OK [HIGHESTMODSEQ 90060128194045007] S: * OK [UNSEEN 12] Message 12 is first unseen S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft \Deleted \Seen \*)] Permanent flags S: A02 OK [READ-WRITE] Sorry, UIDVALIDITY mismatch Modification Sequence and UID Parameters: A server that doesn't support the persistent storage of mod-sequences for the mailbox MUST send the OK untagged response including the NOMODSEQ response code with every successful SELECT or EXAMINE command, as described in [CONDSTORE]. Such a server doesn't need to remember mod-sequences for expunged messages in the mailbox. It MUST ignore the remaining parameters and behave as if no dynamic message data changed. If the provided UIDVALIDITY matches that of the selected mailbox, the server then checks the last known modification sequence. Melnikov, et al. Standards Track [Page 5] RFC 5162 IMAP Quick Mailbox Resync March 2008 The server sends the client any pending flag changes (using FETCH responses that MUST contain UIDs) and expunges those that have occurred in this mailbox since the provided modification sequence. If the list of known UIDs was also provided, the server should only report flag changes and expunges for the specified messages. If the client did not provide the list of UIDs, the server acts as if the client has specified "1:", where is the mailbox's UIDNEXT value minus 1. If the mailbox is empty and never had any messages in it, then lack of the list of UIDs is interpreted as an empty set of UIDs. Thus, the client can process just these pending events and need not perform a full resynchronization. Without the message sequence number matching information, the result of this step is semantically equivalent to the client issuing: tag1 UID FETCH "known-uids" (FLAGS) (CHANGEDSINCE "mod-sequence-value" VANISHED) Example: C: A03 SELECT INBOX (QRESYNC (67890007 90060115194045000 41,43:211,214:541)) S: * OK [CLOSED] S: * 314 EXISTS S: * 15 RECENT S: * OK [UIDVALIDITY 67890007] UIDVALIDITY S: * OK [UIDNEXT 567] Predicted next UID S: * OK [HIGHESTMODSEQ 90060115205545359] S: * OK [UNSEEN 7] There are some unseen messages in the mailbox S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft \Deleted \Seen \*)] Permanent flags S: * VANISHED (EARLIER) 41,43:116,118,120:211,214:540 S: * 49 FETCH (UID 117 FLAGS (\Seen \Answered) MODSEQ (90060115194045001)) S: * 50 FETCH (UID 119 FLAGS (\Draft $MDNSent) MODSEQ (90060115194045308)) S: ... S: * 100 FETCH (UID 541 FLAGS (\Seen $Forwarded) MODSEQ (90060115194045001)) S: A03 OK [READ-WRITE] mailbox selected Message sequence match data: A client MAY provide a parenthesized list of a message sequence set and the corresponding UID sets. Both MUST be provided in ascending order. The server uses this data to restrict the range for which it provides expunged message information. Melnikov, et al. Standards Track [Page 6] RFC 5162 IMAP Quick Mailbox Resync March 2008 Conceptually, the client provides a small sample of sequence numbers for which it knows the corresponding UIDs. The server then compares each sequence number and UID pair the client provides with the current state of the mailbox. If a pair matches, then the client knows of any expunges up to, and including, the message, and thus will not include that range in the VANISHED response, even if the "mod-sequence-value" provided by the client is too old for the server to have data of when those messages were expunged. Thus, if the Nth message number in the first set in the list is 4, and the Nth UID in the second set in the list is 8, and the mailbox's fourth message has UID 8, then no UIDs equal to or less than 8 are present in the VANISHED response. If the (N+1)th message number is 12, and the (N+1)th UID is 24, and the (N+1)th message in the mailbox has UID 25, then the lowest UID included in the VANISHED response would be 9. In the following two examples, the server is unable to remember expunges at all, and only UIDs with messages divisible by three are present in the mailbox. In the first example, the client does not use the fourth parameter; in the second, it provides it. This example is somewhat extreme, but shows that judicious usage of the sequence match data can save a substantial amount of bandwidth. Example: C: A04 SELECT INBOX (QRESYNC (67890007 90060115194045000 1:29997)) S: * 10003 EXISTS S: * 5 RECENT S: * OK [UIDVALIDITY 67890007] UIDVALIDITY S: * OK [UIDNEXT 30013] Predicted next UID S: * OK [HIGHESTMODSEQ 90060115205545359] S: * OK [UNSEEN 7] There are some unseen messages in the mailbox S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft \Deleted \Seen \*)] Permanent flags S: * VANISHED (EARLIER) 1:2,4:5,7:8,10:11,13:14 [...] 29998:29999,30001:30002,30004:30005,30007:30008 S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ (90060115194045027)) S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ (90060115194045028)) S: ... S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ (90060115194045031)) S: A04 OK [READ-WRITE] mailbox selected Melnikov, et al. Standards Track [Page 7] RFC 5162 IMAP Quick Mailbox Resync March 2008 Example: C: B04 SELECT INBOX (QRESYNC (67890007 90060115194045000 1:29997 (5000,7500,9000,9990:9999 15000, 22500,27000,29970,29973,29976,29979,29982,29985,29988,29991, 29994,29997))) S: * 10003 EXISTS S: * 5 RECENT S: * OK [UIDVALIDITY 67890007] UIDVALIDITY S: * OK [UIDNEXT 30013] Predicted next UID S: * OK [HIGHESTMODSEQ 90060115205545359] S: * OK [UNSEEN 7] There are some unseen messages in the mailbox S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft \Deleted \Seen \*)] Permanent flags S: * VANISHED (EARLIER) 29998:29999,30001:30002,30004:30005,30007: 30008 S: * 9889 FETCH (UID 29667 FLAGS (\Seen \Answered) MODSEQ (90060115194045027)) S: * 9890 FETCH (UID 29670 FLAGS (\Draft $MDNSent) MODSEQ (90060115194045028)) S: ... S: * 9999 FETCH (UID 29997 FLAGS (\Seen $Forwarded) MODSEQ (90060115194045031)) S: B04 OK [READ-WRITE] mailbox selected 3.2. VANISHED UID FETCH Modifier [IMAPABNF] has extended the syntax of the FETCH and UID FETCH commands to include an optional FETCH modifier. This document defines a new UID FETCH modifier: VANISHED. Note, that the VANISHED UID FETCH modifier is NOT allowed with a FETCH command. The server MUST return a tagged BAD response if this response is specified as a modifier to the FETCH command. A server MUST respond with a tagged BAD response if the VANISHED UID FETCH modifier is specified and the client hasn't issued "ENABLE QRESYNC" in the current connection. The VANISHED UID FETCH modifier MUST only be specified together with the CHANGEDSINCE UID FETCH modifier. The VANISHED UID FETCH modifier instructs the server to report those messages from the UID set parameter that have been expunged and whose associated mod-sequence is larger than the specified mod-sequence. That is, the client requests to be informed of messages from the specified set that were expunged since the specified mod-sequence. Note that the mod-sequence(s) associated with these messages were Melnikov, et al. Standards Track [Page 8] RFC 5162 IMAP Quick Mailbox Resync March 2008 updated when the messages were expunged (as described above). The expunged messages are reported using the VANISHED response as described in Section 3.6, which MUST contain the EARLIER tag. Any VANISHED (EARLIER) responses MUST be returned before any FETCH responses, as otherwise the client might get confused about how message numbers map to UIDs. Note: A server that receives a mod-sequence smaller than , where is the value of the smallest expunged mod-sequence it remembers minus one, MUST behave as if it was requested to report all expunged messages from the provided UID set parameter. Example 1: Without the VANISHED UID FETCH modifier, a CONDSTORE-aware client [CONDSTORE] needs to issue separate commands to learn of flag changes and expunged messages since the last synchronization: C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345) S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk $AutoJunk $MDNSent)) S: s100 OK FETCH completed C: s101 UID SEARCH 300:500 S: * SEARCH 404 406 407 408 410 412 S: s101 OK search completed Where 300 and 500 are the lowest and highest UIDs from client's cache. The second SEARCH response tells the client that the messages with UIDs 407, 410, and 412 are still present, but their flags haven't changed since the specified modification sequence. Using the VANISHED UID FETCH modifier, it is sufficient to issue only a single command: C: s100 UID FETCH 300:500 (FLAGS) (CHANGEDSINCE 12345 VANISHED) S: * VANISHED (EARLIER) 300:310,405,411 S: * 1 FETCH (UID 404 MODSEQ (65402) FLAGS (\Seen)) S: * 2 FETCH (UID 406 MODSEQ (75403) FLAGS (\Deleted)) S: * 4 FETCH (UID 408 MODSEQ (29738) FLAGS ($NoJunk $AutoJunk $MDNSent)) S: s100 OK FETCH completed Melnikov, et al. Standards Track [Page 9] RFC 5162 IMAP Quick Mailbox Resync March 2008 3.3. EXPUNGE Command Arguments: none Responses: untagged responses: EXPUNGE or VANISHED Result: OK - expunge completed NO - expunge failure: can't expunge (e.g., permission denied) BAD - command unknown or arguments invalid This section updates the definition of the EXPUNGE command described in Section 6.4.3 of [RFC3501]. The EXPUNGE command permanently removes all messages that have the \Deleted flag set from the currently selected mailbox. Before returning an OK to the client, those messages that are removed are reported using a VANISHED response or EXPUNGE responses. If the server is capable of storing modification sequences for the selected mailbox, it MUST increment the per-mailbox mod-sequence if at least one message was permanently removed due to the execution of the EXPUNGE command. For each permanently removed message, the server MUST remember the incremented mod-sequence and corresponding UID. If at least one message got expunged, the server MUST send the updated per-mailbox modification sequence using the HIGHESTMODSEQ response code (defined in [CONDSTORE]) in the tagged OK response. Example: C: A202 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 5 EXPUNGE S: * 8 EXPUNGE S: A202 OK [HIGHESTMODSEQ 20010715194045319] expunged Note: In this example, messages 3, 4, 7, and 11 had the \Deleted flag set. The first "* 3 EXPUNGE" reports message # 3 as expunged. The second "* 3 EXPUNGE" reports message # 4 as expunged (the message number got decremented due to the previous EXPUNGE response). See the description of the EXPUNGE response in [RFC3501] for further explanation. Note that if the server chooses to always send VANISHED responses instead of EXPUNGE responses, the previous example might look like this: Example: C: B202 EXPUNGE S: * VANISHED 405,407,410,425 S: B202 OK [HIGHESTMODSEQ 20010715194045319] expunged Melnikov, et al. Standards Track [Page 10] RFC 5162 IMAP Quick Mailbox Resync March 2008 Here messages with message numbers 3, 4, 7, and 11 have respective UIDs 405, 407, 410, and 425. 3.4. CLOSE Command Arguments: none Responses: no specific responses for this command Result: OK - close completed, now in authenticated state BAD - command unknown or arguments invalid This section updates the definition of the CLOSE command described in Section 6.4.2 of [RFC3501]. The CLOSE command permanently removes all messages that have the \Deleted flag set from the currently selected mailbox, and returns to the authenticated state from the selected state. No untagged EXPUNGE (or VANISHED) responses are sent. If the server is capable of storing modification sequences for the selected mailbox, it MUST increment the per-mailbox mod-sequence if at least one message was permanently removed due to the execution of the CLOSE command. For each permanently removed message, the server MUST remember the incremented mod-sequence and corresponding UID. If at least one message got expunged, the server MUST send the updated per-mailbox modification sequence using the HIGHESTMODSEQ response code (defined in [CONDSTORE]) in the tagged OK response. Example: C: A202 CLOSE S: A202 OK [HIGHESTMODSEQ 20010715194045319] done 3.5. UID EXPUNGE Command Arguments: message set Responses: untagged responses: EXPUNGE or VANISHED Result: OK - expunge completed NO - expunge failure: can't expunge (e.g., permission denied) BAD - command unknown or arguments invalid This section updates the definition of the UID EXPUNGE command described in Section 2.1 of [UIDPLUS]. Servers that implement both [UIDPLUS] and QRESYNC extensions must implement UID EXPUNGE as described in this section. Melnikov, et al. Standards Track [Page 11] RFC 5162 IMAP Quick Mailbox Resync March 2008 The UID EXPUNGE command permanently removes from the currently selected mailbox all messages that both have the \Deleted flag set and have a UID that is included in the specified message set. If a message either does not have the \Deleted flag set or has a UID that is not included in the specified message set, it is not affected. This command is particularly useful for disconnected mode clients. By using UID EXPUNGE instead of EXPUNGE when resynchronizing with the server, the client can avoid inadvertently removing any messages that have been marked as \Deleted by other clients between the time that the client was last connected and the time the client resynchronizes. Before returning an OK to the client, those messages that are removed are reported using a VANISHED response or EXPUNGE responses. If the server is capable of storing modification sequences for the selected mailbox, it MUST increment the per-mailbox mod-sequence if at least one message was permanently removed due to the execution of the UID EXPUNGE command. For each permanently removed message, the server MUST remember the incremented mod-sequence and corresponding UID. If at least one message got expunged, the server MUST send the updated per-mailbox modification sequence using the HIGHESTMODSEQ response code (defined in [CONDSTORE]) in the tagged OK response. Example: C: . UID EXPUNGE 3000:3002 S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: . OK [HIGHESTMODSEQ 20010715194045319] Ok Note: In this example, at least messages with message numbers 3, 4, and 5 (UIDs 3000 to 3002) had the \Deleted flag set. The first "* 3 EXPUNGE" reports message # 3 as expunged. The second "* 3 EXPUNGE" reports message # 4 as expunged (the message number got decremented due to the previous EXPUNGE response). See the description of the EXPUNGE response in [RFC3501] for further explanation. 3.6. VANISHED Response Contents: an optional EARLIER tag list of UIDs The VANISHED response reports that the specified UIDs have been permanently removed from the mailbox. This response is similar to the EXPUNGE response [RFC3501]; however, it can return information about multiple messages, and it returns UIDs instead of message Melnikov, et al. Standards Track [Page 12] RFC 5162 IMAP Quick Mailbox Resync March 2008 numbers. The first benefit saves bandwidth, while the second is more convenient for clients that only use UIDs to access the IMAP server. The VANISHED response has the same restrictions on when it can be sent as does the EXPUNGE response (see below). The VANISHED response has two forms. The first form contains the EARLIER tag, which signifies that the response was caused by a UID FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) command. This response is sent if the UID set parameter to the UID FETCH (VANISHED) command includes UIDs of messages that are no longer in the mailbox. When the client sees a VANISHED EARLIER response, it MUST NOT decrement message sequence numbers for each successive message in the mailbox. The second form doesn't contain the EARLIER tag and is described below. Once a client has issued "ENABLE QRESYNC", the server SHOULD use the VANISHED response without the EARLIER tag instead of the EXPUNGE response. The server SHOULD continue using VANISHED in lieu of EXPUNGE for the duration of the connection. In particular, this affects the EXPUNGE [RFC3501] and UID EXPUNGE [UIDPLUS] commands, as well as messages expunged in other connections. Such a VANISHED response MUST NOT contain the EARLIER tag. A VANISHED response sent because of an EXPUNGE or UID EXPUNGE command or because messages were expunged in other connections (i.e., the VANISHED response without the EARLIER tag) also decrements the number of messages in the mailbox; it is not necessary for the server to send an EXISTS response with the new value. It also decrements message sequence numbers for each successive message in the mailbox (see the example at the end of this section). Note that a VANISHED response caused by EXPUNGE, UID EXPUNGE, or messages expunged in other connections SHOULD only contain UIDs for messages expunged since the last VANISHED/EXPUNGE response sent for the currently opened mailbox or since the mailbox was opened. That is, servers SHOULD NOT send UIDs for previously expunged messages, unless explicitly requested to do so by the UID FETCH (VANISHED) command. Note that client implementors must take care to properly decrement the number of messages in the mailbox even if a server violates this last SHOULD or repeats the same UID multiple times in the returned UID set. In general, this means that a client using this extension should either avoid using message numbers entirely, or have a complete mapping of UIDs to message sequence numbers for the selected mailbox. Melnikov, et al. Standards Track [Page 13] RFC 5162 IMAP Quick Mailbox Resync March 2008 Because clients handle the two different forms of the VANISHED response differently, servers MUST NOT report UIDs resulting from a UID FETCH (VANISHED) or a SELECT/EXAMINE (QRESYNC) in the same VANISHED response as UIDs of messages expunged now (i.e., messages expunged in other connections). Instead, the server MUST send separate VANISHED responses: one with the EARLIER tag and one without. A VANISHED response MUST NOT be sent when no command is in progress, nor while responding to a FETCH, STORE, or SEARCH command. This rule is necessary to prevent a loss of synchronization of message sequence numbers between client and server. A command is not "in progress" until the complete command has been received; in particular, a command is not "in progress" during the negotiation of command continuation. Note: UID FETCH, UID STORE, and UID SEARCH are different commands from FETCH, STORE, and SEARCH. A VANISHED response MAY be sent during a UID command. However, the VANISHED response MUST NOT be sent during a UID SEARCH command that contains message numbers in the search criteria. The update from the VANISHED response MUST be recorded by the client. Example: Let's assume that there is the following mapping between message numbers and UIDs in the currently selected mailbox (here "X" marks messages with the \Deleted flag set, and "x" represents UIDs which are not relevant for the example): Message numbers: 1 2 3 4 5 6 7 8 9 10 11 UIDs: x 504 505 507 508 x 510 x x x 625 \Deleted messages: X X X X In the presence of the extension defined in this document: C: A202 EXPUNGE S: * VANISHED 505,507,510,625 S: A202 OK EXPUNGE completed Without the QRESYNC extension, the same example might look like: C: A202 EXPUNGE S: * 3 EXPUNGE S: * 3 EXPUNGE S: * 5 EXPUNGE S: * 8 EXPUNGE S: A202 OK EXPUNGE completed Melnikov, et al. Standards Track [Page 14] RFC 5162 IMAP Quick Mailbox Resync March 2008 (Continuing previous example) If subsequently messages with UIDs 504 and 508 got marked as \Deleted: C: A210 EXPUNGE S: * VANISHED 504,508 S: A210 OK EXPUNGE completed i.e., the last VANISHED response only contains UIDs of messages expunged since the previous VANISHED response. 3.7. CLOSED Response Code The CLOSED response code has no parameters. A server implementing the extension defined in this document MUST return the CLOSED response code when the currently selected mailbox is closed implicitly using the SELECT/EXAMINE command on another mailbox. The CLOSED response code serves as a boundary between responses for the previously opened mailbox (which was closed) and the newly selected mailbox: all responses before the CLOSED response code relate to the mailbox that was closed, and all subsequent responses relate to the newly opened mailbox. There is no need to return the CLOSED response code on completion of the CLOSE or the UNSELECT [UNSELECT] command (or similar) whose purpose is to close the currently selected mailbox without opening a new one. 4. Server Implementation Considerations This section describes a minimalist implementation, a moderate implementation, and an example of a full implementation. 4.1. Server Implementations That Don't Store Extra State Strictly speaking, a server implementation that doesn't remember mod- sequences associated with expunged messages can be considered compliant with this specification. Such implementations return all expunged messages specified in the UID set of the UID FETCH (VANISHED) command every time, without paying attention to the specified CHANGEDSINCE mod-sequence. Such implementations are discouraged, as they can end up returning VANISHED responses that are bigger than the result of a UID SEARCH command for the same UID set. Clients that use the message sequence match data can reduce the scope of this VANISHED response substantially in the typical case where expunges have not happened, or happen only toward the end of the mailbox. Melnikov, et al. Standards Track [Page 15] RFC 5162 IMAP Quick Mailbox Resync March 2008 4.2. Server Implementations Storing Minimal State A server that stores the HIGHESTMODSEQ value at the time of the last EXPUNGE can omit the VANISHED response when a client provides a MODSEQ value that is equal to, or higher than, the current value of this datum, that is, when there have been no EXPUNGEs. A client providing message sequence match data can reduce the scope as above. In the case where there have been no expunges, the server can ignore this data. 4.3. Additional State Required on the Server When compared to the [CONDSTORE] extension, this extension requires servers to store additional state associated with expunged messages. Note that implementations are not required to store this state in persistent storage; however, use of persistent storage is advisable. One possible way to correctly implement the extension described in this document is to store a queue of pairs. can be represented as a sequence of pairs. When messages are expunged, one or more entries are added to the queue tail. When the server receives a request to return messages expunged since a given mod-sequence, it will search the queue from the tail (i.e., going from the highest expunged mod-sequence to the lowest) until it sees the first record with a mod-sequence less than or equal to the given mod-sequence or it reaches the head of the queue. Note that indefinitely storing information about expunged messages can cause storage and related problems for an implementation. In the worst case, this could result in almost 64Gb of storage for each IMAP mailbox. For example, consider an implementation that stores triples for each range of messages expunged at the same time. Each triple requires 16 octets: 4 octets for each of the two UIDs, and 8 octets for the mod-sequence. Assume that there is a mailbox containing a single message with a UID of 2**32-1 (the maximum possible UID value), where messages had previously existed with UIDs starting at 1, and have been expunged one at a time. For this mailbox alone, storage is required for the triples <1, 1, modseq1>, <2, 2, modseq2>, ..., <2**32-2, 2**32-2, modseq4294967294>. Melnikov, et al. Standards Track [Page 16] RFC 5162 IMAP Quick Mailbox Resync March 2008 Hence, implementations are encouraged to adopt strategies to protect against such storage problems, such as limiting the size of the queue used to store mod-sequences for expunged messages and "expiring" older records when this limit is reached. When the selected implementation-specific queue limit is reached, the oldest record(s) are deleted from the queue (note that such records are located at the queue head). For all such "expired" records, the server needs to store a single mod-sequence, which is the highest mod-sequence for all "expired" expunged messages. Note that if the client provides the message sequence match data, this can heavily reduce the data cost of sending a complete set of missing UIDs; thus, reducing the problems for clients if a server is unable to persist much of this queue. If the queue contains data back to the requested mod-sequence, this data can be ignored. Also, note that if the UIDVALIDITY of the mailbox changes or if the mailbox is deleted, then any state associated with expunged messages doesn't need to be preserved and SHOULD be deleted. 5. Updated Synchronization Sequence This section updates the description of optimized synchronization in Section 6.1 of the [IMAP-DISC]. An advanced disconnected mail client should use the QRESYNC and [CONDSTORE] extensions when they are supported by the server. The client uses the value from the HIGHESTMODSEQ OK response code received on mailbox opening to determine if it needs to resynchronize. Once the synchronization is complete, it MUST cache the received value (unless the mailbox UIDVALIDITY value has changed; see below). The client MUST update its copy of the HIGHESTMODSEQ value whenever the server sends a subsequent HIGHESTMODSEQ OK response code. After completing a full synchronization, the client MUST also take note of any unsolicited MODSEQ FETCH data items received from the server. Whenever the client receives a tagged response to a command, it calculates the highest value among all MODSEQ FETCH data items received since the last tagged response. If this value is bigger than the client's copy of the HIGHESTMODSEQ value, then the client MUST use this value as its new HIGHESTMODSEQ value. Note: It is not safe to update the client's copy of the HIGHESTMODSEQ value with a MODSEQ FETCH data item value as soon as it is received because servers are not required to send MODSEQ FETCH data items in increasing modseqence order. This can lead to the client missing some changes in case of connectivity loss. Melnikov, et al. Standards Track [Page 17] RFC 5162 IMAP Quick Mailbox Resync March 2008 When opening the mailbox for synchronization, the client uses the QRESYNC parameter to the SELECT/EXAMINE command. The QRESYNC parameter is followed by the UIDVALIDITY and mailbox HIGHESTMODSEQ values, as known to the client. It can be optionally followed by the set of UIDs, for example, if the client is only interested in partial synchronization of the mailbox. The client may also transmit a list containing its knowledge of message numbers. If the SELECT/EXAMINE command is successful, the client compares UIDVALIDITY as described in step d)1) in Section 3 of the [IMAP-DISC]. If the cached UIDVALIDITY value matches the one returned by the server and the server also returns the HIGHESTMODSEQ response code, then the server reports expunged messages and returns flag changes for all messages specified by the client in the UID set parameter (or for all messages in the mailbox, if the client omitted the UID set parameter). At this point, the client is synchronized, except for maybe the new messages. If upon a successful SELECT/EXAMINE (QRESYNC) command the client receives a NOMODSEQ OK untagged response (instead of the HIGHESTMODSEQ response code), it MUST remove the last known HIGHESTMODSEQ value from its cache and follow the more general instructions in Section 3 of the [IMAP-DISC]. At this point, the client is in sync with the server regarding old messages. This client can now fetch information about new messages (if requested by the user). Step d) ("Server-to-client synchronization") in Section 4 of the [IMAP-DISC] in the presence of the QRESYNC & CONDSTORE extensions is amended as follows: d) "Server-to-client synchronization" -- for each mailbox that requires synchronization, do the following: 1a) Check the mailbox UIDVALIDITY (see Section 4.1 of the [IMAP-DISC] for more details) after issuing SELECT/EXAMINE (QRESYNC) command. If the UIDVALIDITY value returned by the server differs, the client MUST * empty the local cache of that mailbox; * "forget" the cached HIGHESTMODSEQ value for the mailbox; Melnikov, et al. Standards Track [Page 18] RFC 5162 IMAP Quick Mailbox Resync March 2008 * remove any pending "actions" which refer to UIDs in that mailbox. Note, this doesn't affect actions performed on client generated fake UIDs (see Section 5 of the [IMAP-DISC]); 2) Fetch the current "descriptors"; I) Discover new messages. 3) Fetch the bodies of any "interesting" messages that the client doesn't already have. Example: The UIDVALIDITY value is the same, but the HIGHESTMODSEQ value has changed on the server while the client was offline: C: A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198)) S: * 172 EXISTS S: * 1 RECENT S: * OK [UNSEEN 12] Message 12 is first unseen S: * OK [UIDVALIDITY 3857529045] UIDs valid S: * OK [UIDNEXT 201] Predicted next UID S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft) S: * OK [PERMANENTFLAGS (\Deleted \Seen \*)] Limited S: * OK [HIGHESTMODSEQ 20010715194045007] S: * VANISHED (EARLIER) 1:5,7:8,10:15 S: * 2 FETCH (UID 6 MODSEQ (20010715205008000) FLAGS (\Deleted)) S: * 5 FETCH (UID 9 MODSEQ (20010715195517000) FLAGS ($NoJunk $AutoJunk $MDNSent)) ... S: A142 OK [READ-WRITE] SELECT completed 6. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. Non-terminals referenced but not defined below are as defined by [RFC3501], [CONDSTORE], or [IMAPABNF]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. Melnikov, et al. Standards Track [Page 19] RFC 5162 IMAP Quick Mailbox Resync March 2008 capability =/ "QRESYNC" select-param = "QRESYNC" SP "(" uidvalidity SP mod-sequence-value [SP known-uids] [SP seq-match-data] ")" ;; conforms to the generic select-param ;; syntax defined in [IMAPABNF] seq-match-data = "(" known-sequence-set SP known-uid-set ")" uidvalidity = nz-number known-uids = sequence-set ;; sequence of UIDs, "*" is not allowed known-sequence-set = sequence-set ;; set of message numbers corresponding to ;; the UIDs in known-uid-set, in ascending order. ;; * is not allowed. known-uid-set = sequence-set ;; set of UIDs corresponding to the messages in ;; known-sequence-set, in ascending order. ;; * is not allowed. message-data =/ expunged-resp expunged-resp = "VANISHED" [SP "(EARLIER)"] SP known-uids rexpunges-fetch-mod = "VANISHED" ;; VANISHED UID FETCH modifier conforms ;; to the fetch-modifier syntax ;; defined in [IMAPABNF]. It is only ;; allowed in the UID FETCH command. resp-text-code =/ "CLOSED" 7. Security Considerations As always, it is important to thoroughly test clients and servers implementing this extension, as it changes how the server reports expunged messages to the client. Security considerations relevant to [CONDSTORE] are relevant to this extension. This document doesn't raise any new security concerns not already raised by [CONDSTORE] or [RFC3501]. Melnikov, et al. Standards Track [Page 20] RFC 5162 IMAP Quick Mailbox Resync March 2008 8. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG approved experimental RFC. The registry is currently located at: http://www.iana.org/assignments/imap4-capabilities This document defines the QRESYNC IMAP capability. IANA has added this capability to the registry. 9. Acknowledgments Thanks to Steve Hole, Cyrus Daboo, and Michael Wener for encouraging creation of this document. Valuable comments, both in agreement and in dissent, were received from Timo Sirainen, Michael Wener, Randall Gellens, Arnt Gulbrandsen, Chris Newman, Peter Coates, Mark Crispin, Elwyn Davies, Dan Karp, Eric Rescorla, and Mike Zraly. This document takes substantial text from [RFC3501] by Mark Crispin. 10. References 10.1. Normative References [ABNF] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [CONDSTORE] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. [ENABLE] Gulbrandsen, A., Ed. and A. Melnikov, Ed., "The IMAP ENABLE Extension", RFC 5161, March 2008. [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS extension", RFC 4315, December 2005. Melnikov, et al. Standards Track [Page 21] RFC 5162 IMAP Quick Mailbox Resync March 2008 10.2. Informative References [IMAP-DISC] Melnikov, A., Ed., "Synchronization Operations For Disconnected Imap4 Clients", RFC 4549, June 2006. [UNSELECT] Melnikov, A., "Internet Message Access Protocol (IMAP) UNSELECT command", RFC 3691, February 2004. Authors' Addresses Alexey Melnikov Isode Ltd 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com Dave Cridland Isode Ltd 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: dave.cridland@isode.com Corby Wilson Nokia 5 Wayside Rd. Burlington, MA 01803 USA EMail: corby@computer.org Melnikov, et al. Standards Track [Page 22] RFC 5162 IMAP Quick Mailbox Resync March 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Melnikov, et al. Standards Track [Page 23] ================================================ FILE: docs/rfcs/rfc5182.IMAP_extension_last_SEARCH_result.txt ================================================ Network Working Group A. Melnikov Request for Comments: 5182 Isode Ltd. Updates: 3501 March 2008 Category: Standards Track IMAP Extension for Referencing the Last SEARCH Result Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract Many IMAP clients use the result of a SEARCH command as the input to perform another operation, for example, fetching the found messages, deleting them, or copying them to another mailbox. This can be achieved using standard IMAP operations described in RFC 3501; however, this would be suboptimal. The server will send the list of found messages to the client; after that, the client will have to parse the list, reformat it, and send it back to the server. The client can't pipeline the SEARCH command with the subsequent command, and, as a result, the server might not be able to perform some optimizations. This document proposes an IMAP extension that allows a client to tell a server to use the result of a SEARCH (or Unique Identifier (UID) SEARCH) command as an input to any subsequent command. 1. Introduction Many IMAP clients use the result of a SEARCH command as the input to perform another operation, for example, fetching the found messages, deleting them, or copying them to another mailbox. This document proposes an IMAP extension that allows a client to tell a server to use the result of a SEARCH (or UID SEARCH) command as an input to any subsequent command. The SEARCH result reference extension defines a new SEARCH result option [IMAPABNF] "SAVE" that tells the server to remember the result of the SEARCH or UID SEARCH command (as well as any command based on SEARCH, e.g., SORT and THREAD [SORT]) and store it in an internal Melnikov Standards Track [Page 1] RFC 5182 Last SEARCH Result Reference March 2008 variable that we will reference as the "search result variable". The client can use the "$" marker to reference the content of this internal variable. The "$" marker can be used instead of message sequence or UID sequence in order to indicate that the server should substitute it with the list of messages from the search result variable. Thus, the client can use the result of the latest remembered SEARCH command as a parameter to another command. The search result marker has several advantages: * it avoids wasted bandwidth and associated delay; * it allows the client to pipeline a SEARCH [IMAP4] command with a subsequent FETCH/STORE/COPY/SEARCH [IMAP4] or UID EXPUNGE [UIDPLUS] command; * the client doesn't need to spend time reformatting the result of a SEARCH command into a message set used in the subsequent command; * it allows the server to perform optimizations. For example, if the server can execute several pipelined commands in parallel (or out of order), presence of the search result marker can allow the server to decide which commands may or may not be executed out of order. In absence of any other SEARCH result option, the SAVE result option also suppresses any SEARCH response that would have been otherwise returned by the SEARCH command. 1.1. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. Explanatory comments in examples start with // and are not part of the protocol. Melnikov Standards Track [Page 2] RFC 5182 Last SEARCH Result Reference March 2008 2. Overview 2.1. Normative Description of the SEARCHRES Extension The SEARCH result reference extension described in this document is present in any IMAP4 server implementation that returns "SEARCHRES" as one of the supported capabilities in the CAPABILITY command response. Any such server MUST also implement the [ESEARCH] extension. Upon successful completion of a SELECT or an EXAMINE command (after the tagged OK response), the current search result variable is reset to the empty sequence. A successful SEARCH command with the SAVE result option sets the value of the search result variable to the list of messages found in the SEARCH command. For example, if no messages were found, the search result variable will contain the empty list. Any of the following SEARCH commands MUST NOT change the search result variable: o a SEARCH command that caused the server to return the BAD tagged response, o a SEARCH command with no SAVE result option that caused the server to return NO tagged response, o a successful SEARCH command with no SAVE result option. A SEARCH command with the SAVE result option that caused the server to return the NO tagged response sets the value of the search result variable to the empty sequence. When a message listed in the search result variable is EXPUNGEd, it is automatically removed from the list. Implementors are reminded that if the server stores the list as a list of message numbers, it MUST automatically adjust them when notifying the client about expunged messages, as described in Section 7.4.1 of [IMAP4]. If the server decides to send a new UIDVALIDITY value while the mailbox is opened, this causes resetting of the search variable to the empty list. Melnikov Standards Track [Page 3] RFC 5182 Last SEARCH Result Reference March 2008 Note that even if the "$" marker contains the empty list of messages, it must be treated by all commands accepting message sets as parameters as a valid, but non-matching list of messages. For example, the "FETCH $" command would return a tagged OK response and no FETCH responses. See also the Example 5 below. Note that even if the "$" marker contains the empty list of messages, it must be treated as a valid but non-matching list of messages, by all commands that accept message sets as parameters. Implementation note: server implementors should note that "$" can reference IMAP message sequences or UID sequences, depending on the context where it is used. For example, the "$" marker can be set as a result of a SEARCH (SAVE) command and used as a parameter to a UID FETCH command (which accepts a UID sequence, not a message sequence), or the "$" marker can be set as a result of a UID SEARCH (SAVE) command and used as a parameter to a FETCH command (which accepts a message sequence, not a UID sequence). 2.2. Examples 1) The following example demonstrates how the client can use the result of a SEARCH command to FETCH headers of interesting messages: Example 1: C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" S: A282 OK SEARCH completed, result saved C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) S: * 2 FETCH (UID 14 ... S: * 84 FETCH (UID 100 ... S: * 882 FETCH (UID 1115 ... S: A283 OK completed The client can also pipeline the two commands: Example 2: C: A282 SEARCH RETURN (SAVE) FLAGGED SINCE 1-Feb-1994 NOT FROM "Smith" C: A283 FETCH $ (UID INTERNALDATE FLAGS RFC822.HEADER) S: A282 OK SEARCH completed S: * 2 FETCH (UID 14 ... S: * 84 FETCH (UID 100 ... S: * 882 FETCH (UID 1115 ... S: A283 OK completed Melnikov Standards Track [Page 4] RFC 5182 Last SEARCH Result Reference March 2008 2) The following example demonstrates that the result of one SEARCH command can be used as input to another SEARCH command: Example 3: C: A300 SEARCH RETURN (SAVE) SINCE 1-Jan-2004 NOT FROM "Smith" S: A300 OK SEARCH completed C: A301 UID SEARCH UID $ SMALLER 4096 S: * SEARCH 17 900 901 S: A301 OK completed Note that the second command in Example 3 can be replaced with: C: A301 UID SEARCH $ SMALLER 4096 and the result of the command would be the same. 3) The following example shows that the "$" marker can be combined with other message numbers using the OR SEARCH criterion. Example 4: C: P282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 NOT FROM "Smith" S: P282 OK SEARCH completed C: P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) TEXT {8} C: YYYYYYYY S: * SEARCH 882 1102 3003 3005 3006 S: P283 OK completed Note: Since this document format is restricted to 7-bit ASCII text, it is not possible to show actual UTF-8 data. The "YYYYYYYY" is a placeholder for what would be 8 octets of 8-bit data in an actual transaction. Melnikov Standards Track [Page 5] RFC 5182 Last SEARCH Result Reference March 2008 4) The following example demonstrates that a failed SEARCH sets the search result variable to the empty list. Example 5: C: B282 SEARCH RETURN (SAVE) SINCE 1-Feb-1994 NOT FROM "Smith" S: B282 OK SEARCH completed C: B283 SEARCH CHARSET KOI8-R (OR $ 1,3000:3021) TEXT {4} C: XXXX S: B283 NO [BADCHARSET UTF-8] KOI8-R is not supported //After this command the saved result variable contains //no messages. A client that wants to reissue the B283 //SEARCH command with another CHARSET would have to reissue //the B282 command as well. One possible workaround for //this is to include the desired CHARSET parameter //in the earliest SEARCH RETURN (SAVE) command in a //sequence of related SEARCH commands. //A better approach might be to always use CHARSET UTF-8 //instead. Note: Since this document format is restricted to 7-bit ASCII text, it is not possible to show actual KOI8-R data. The "XXXX" is a placeholder for what would be 4 octets of 8-bit data in an actual transaction. 5) The following example demonstrates that it is not an error to use the "$" marker when it contains no messages. Example 6: C: E282 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 NOT FROM "Eric" C: E283 COPY $ "Other Messages" //The "$" contains no messages S: E282 OK SEARCH completed S: E283 OK COPY completed, nothing copied 2.3. Multiple Commands in Progress Use of a SEARCH RETURN (SAVE) command followed by a command using the "$" marker creates direct dependency between the two commands. As directed by Section 5.5 of [IMAP4], a server MUST execute the two commands in the order they were received. (A server capable of out-of-order execution can in some cases execute the two commands in parallel, for example, if a SEARCH RETURN (SAVE) is followed by "SEARCH $", the search criteria from the first command can be directly substituted into the second command.) Melnikov Standards Track [Page 6] RFC 5182 Last SEARCH Result Reference March 2008 A client supporting this extension MAY pipeline a SEARCH RETURN (SAVE) command with one or more command using the "$" marker, as long as this doesn't create an ambiguity, as described in Section 5.5 of [IMAP4]. Example 7: C: F282 SEARCH RETURN (SAVE) KEYWORD $Junk C: F283 COPY $ "Junk" C: F284 STORE $ +FLAGS.Silent (\Deleted) S: F282 OK SEARCH completed S: F283 OK COPY completed S: F284 OK STORE completed Example 8: C: G282 SEARCH RETURN (SAVE) KEYWORD $Junk C: G283 SEARCH RETURN (ALL) SINCE 28-Oct-2006 FROM "Eric" //The server can execute the two SEARCH commands //in any order, as they don't have any dependency. //Note that the second command is making use of //the [ESEARCH] extension. S: * ESEARCH (TAG "G283") ALL 3:15,27,29:103 S: G283 OK SEARCH completed S: G282 OK SEARCH completed The following example demonstrates that the result of the second SEARCH always overrides the result of the first. Example 9: C: H282 SEARCH RETURN (SAVE) KEYWORD $Junk C: H283 SEARCH RETURN (SAVE) SINCE 28-Oct-2006 FROM "Eric" S: H282 OK SEARCH completed S: H283 OK SEARCH completed 2.4. Interaction with ESEARCH Extension Servers that implement the extension defined in this document MUST implement [ESEARCH] and conform to additional requirements listed in this section. The SAVE result option doesn't change whether the server would return items corresponding to MIN, MAX, ALL, or COUNT [ESEARCH] result options. Melnikov Standards Track [Page 7] RFC 5182 Last SEARCH Result Reference March 2008 When the SAVE result option is combined with the MIN or MAX [ESEARCH] result option, and none of the other ESEARCH result options are present, the corresponding MIN/MAX is returned (if the search result is not empty), but the "$" marker would contain a single message as returned in the MIN/MAX return item. If the SAVE result option is combined with both MIN and MAX result options, and none of the other ESEARCH result options are present, the "$" marker would contain one or two messages as returned in the MIN/MAX return items. If the SAVE result option is combined with the ALL and/or COUNT result option(s), the "$" marker would always contain all messages found by the SEARCH or UID SEARCH command. (Note that the last rule might affect ESEARCH implementations that optimize how the COUNT result is constructed.) The following table summarizes the additional requirement on ESEARCH server implementations described in this section. +----------------+-------------------+ | Combination of | "$" marker value | | Result option | | +----------------+-------------------+ | SAVE MIN | MIN | +----------------+-------------------+ | SAVE MAX | MAX | +----------------+-------------------+ | SAVE MIN MAX | MIN & MAX | +----------------+-------------------+ | SAVE * [m] | all found messages| +----------------+-------------------+ where '*' means "ALL" and/or "COUNT" '[m]' means optional "MIN" and/or "MAX" Melnikov Standards Track [Page 8] RFC 5182 Last SEARCH Result Reference March 2008 The following example demonstrates behavioral difference for different combinations of ESEARCH result options. Explanatory comments start with // and are not part of the protocol: Example 10: C: C282 SEARCH RETURN (ALL) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C283") ALL 2,10:15,21 //$ value hasn't changed S: C282 OK SEARCH completed C: C283 SEARCH RETURN (ALL SAVE) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C283") ALL 2,10:15,21 //$ value is 2,10:15,21 S: C283 OK SEARCH completed C: C284 SEARCH RETURN (SAVE MIN) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C284") MIN 2 //$ value is 2 S: C284 OK SEARCH completed C: C285 SEARCH RETURN (MAX SAVE MIN) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C285") MIN 2 MAX 21 //$ value is 2,21 S: C285 OK SEARCH completed C: C286 SEARCH RETURN (MAX SAVE MIN COUNT) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C286") MIN 2 MAX 21 COUNT 8 //$ value is 2,10:15,21 S: C286 OK SEARCH completed C: C286 SEARCH RETURN (ALL SAVE MIN) SINCE 12-Feb-2006 NOT FROM "Smith" S: * ESEARCH (TAG "C286") MIN 2 ALL 2,10:15,21 //$ value is 2,10:15,21 S: C286 OK SEARCH completed Melnikov Standards Track [Page 9] RFC 5182 Last SEARCH Result Reference March 2008 2.5. Refusing to Save Search Results In some cases, the server MAY refuse to save a SEARCH (SAVE) result, for example, if an internal limit on the number of saved results is reached. In this case, the server MUST return a tagged NO response containing the NOTSAVED response code and set the search result variable to the empty sequence, as described in Section 2.1. 3. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [ABNF]. Non-terminals referenced but not defined below are as defined in [IMAP4] or [IMAPABNF]. Except as noted otherwise, all alphabetic characters are case-insensitive. The use of upper- or lower-case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. capability =/ "SEARCHRES" ;; capability is defined in [IMAP4] sequence-set =/ seq-last-command ;; extends sequence-set to allow for ;; "result of the last command" indicator. seq-last-command = "$" search-return-opt = "SAVE" ;; conforms to generic search-return-opt ;; syntax defined in [IMAPABNF] resp-text-code =/ "NOTSAVED" ;; from [IMAP4] Melnikov Standards Track [Page 10] RFC 5182 Last SEARCH Result Reference March 2008 4. Security Considerations This extension requires the server to keep additional state, that may be used to simplify Denial of Service attacks. In order to minimize damage from such attacks, server implementations MAY limit the number of saved searches they allow across all connections at any given time and return the tagged NO response containing the NOTSAVED response code (see Section 2.5) to a SEARCH RETURN (SAVE) command when this limit is exceeded. Apart from that, it is believed that this extension doesn't raise any additional security concerns not already discussed in [IMAP4]. 5. IANA Considerations This document defines the "SEARCHRES" IMAP capability. IANA has added it to the IMAP4 Capabilities Registry, which is currently located at: http://www.iana.org/assignments/imap4-capabilities 6. Acknowledgments The author would like to thank Mark Crispin, Cyrus Daboo, and Curtis King for remembering that this document had to be written, as well as for comments and corrections received. The author would also like to thank Dave Cridland, Mark Crispin, Chris Newman, Dan Karp, and Spencer Dawkins for comments and corrections received. Valuable comments, both in agreement and in dissent, were received from Arnt Gulbrandsen. Melnikov Standards Track [Page 11] RFC 5182 Last SEARCH Result Reference March 2008 7. References 7.1. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [IMAP4] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [IMAPABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH Command for Controlling What Kind of Information Is Returned", RFC 4731, November 2006. 7.2. Informative References [UIDPLUS] Crispin, M., "Internet Message Access Protocol (IMAP) - UIDPLUS extension", RFC 4315, December 2005. [SORT] Crispin, M. and K. Murchison, "INTERNET MESSAGE ACCESS PROTOCOL - SORT AND THREAD EXTENSIONS", Work in Progress, Septemeber 2007. Author's Address Alexey Melnikov Isode Ltd. 5 Castle Business Village, 36 Station Road, Hampton, Middlesex, TW12 2BX, United Kingdom EMail: Alexey.Melnikov@isode.com Melnikov Standards Track [Page 12] RFC 5182 Last SEARCH Result Reference March 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Melnikov Standards Track [Page 13] ================================================ FILE: docs/rfcs/rfc5182.Sieve_and_extensions.txt ================================================ Network Working Group P. Guenther, Ed. Request for Comments: 5228 Sendmail, Inc. Obsoletes: 3028 T. Showalter, Ed. Category: Standards Track January 2008 Sieve: An Email Filtering Language Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract This document describes a language for filtering email messages at time of final delivery. It is designed to be implementable on either a mail client or mail server. It is meant to be extensible, simple, and independent of access protocol, mail architecture, and operating system. It is suitable for running on a mail server where users may not be allowed to execute arbitrary programs, such as on black box Internet Message Access Protocol (IMAP) servers, as the base language has no variables, loops, or ability to shell out to external programs. Guenther & Showalter Standards Track [Page 1] RFC 5228 Sieve: An Email Filtering Language January 2008 Table of Contents 1. Introduction ....................................................4 1.1. Conventions Used in This Document ..........................4 1.2. Example Mail Messages ......................................5 2. Design ..........................................................6 2.1. Form of the Language .......................................6 2.2. Whitespace .................................................7 2.3. Comments ...................................................7 2.4. Literal Data ...............................................7 2.4.1. Numbers .............................................7 2.4.2. Strings .............................................8 2.4.2.1. String Lists ...............................9 2.4.2.2. Headers ....................................9 2.4.2.3. Addresses .................................10 2.4.2.4. Encoding Characters Using "encoded-character" .......................10 2.5. Tests .....................................................11 2.5.1. Test Lists .........................................12 2.6. Arguments .................................................12 2.6.1. Positional Arguments ...............................12 2.6.2. Tagged Arguments ...................................12 2.6.3. Optional Arguments .................................13 2.6.4. Types of Arguments .................................13 2.7. String Comparison .........................................13 2.7.1. Match Type .........................................14 2.7.2. Comparisons across Character Sets ..................15 2.7.3. Comparators ........................................15 2.7.4. Comparisons against Addresses ......................16 2.8. Blocks ....................................................17 2.9. Commands ..................................................17 2.10. Evaluation ...............................................18 2.10.1. Action Interaction ................................18 2.10.2. Implicit Keep .....................................18 2.10.3. Message Uniqueness in a Mailbox ...................19 2.10.4. Limits on Numbers of Actions ......................19 2.10.5. Extensions and Optional Features ..................19 2.10.6. Errors ............................................20 2.10.7. Limits on Execution ...............................20 3. Control Commands ...............................................21 3.1. Control if ................................................21 3.2. Control require ...........................................22 3.3. Control stop ..............................................22 4. Action Commands ................................................23 4.1. Action fileinto ...........................................23 4.2. Action redirect ...........................................23 4.3. Action keep ...............................................24 4.4. Action discard ............................................25 Guenther & Showalter Standards Track [Page 2] RFC 5228 Sieve: An Email Filtering Language January 2008 5. Test Commands ..................................................26 5.1. Test address ..............................................26 5.2. Test allof ................................................27 5.3. Test anyof ................................................27 5.4. Test envelope .............................................27 5.5. Test exists ...............................................28 5.6. Test false ................................................28 5.7. Test header ...............................................29 5.8. Test not ..................................................29 5.9. Test size .................................................29 5.10. Test true ................................................30 6. Extensibility ..................................................30 6.1. Capability String .........................................31 6.2. IANA Considerations .......................................31 6.2.1. Template for Capability Registrations ..............32 6.2.2. Handling of Existing Capability Registrations ......32 6.2.3. Initial Capability Registrations ...................32 6.3. Capability Transport ......................................33 7. Transmission ...................................................33 8. Parsing ........................................................34 8.1. Lexical Tokens ............................................34 8.2. Grammar ...................................................36 8.3. Statement Elements ........................................36 9. Extended Example ...............................................37 10. Security Considerations .......................................38 11. Acknowledgments ...............................................39 12. Normative References ..........................................39 13. Informative References ........................................40 14. Changes from RFC 3028 .........................................41 Guenther & Showalter Standards Track [Page 3] RFC 5228 Sieve: An Email Filtering Language January 2008 1. Introduction This memo documents a language that can be used to create filters for electronic mail. It is not tied to any particular operating system or mail architecture. It requires the use of [IMAIL]-compliant messages, but should otherwise generalize to many systems. The language is powerful enough to be useful but limited in order to allow for a safe server-side filtering system. The intention is to make it impossible for users to do anything more complex (and dangerous) than write simple mail filters, along with facilitating the use of graphical user interfaces (GUIs) for filter creation and manipulation. The base language was not designed to be Turing- complete: it does not have a loop control structure or functions. Scripts written in Sieve are executed during final delivery, when the message is moved to the user-accessible mailbox. In systems where the Mail Transfer Agent (MTA) does final delivery, such as traditional Unix mail, it is reasonable to filter when the MTA deposits mail into the user's mailbox. There are a number of reasons to use a filtering system. Mail traffic for most users has been increasing due to increased usage of email, the emergence of unsolicited email as a form of advertising, and increased usage of mailing lists. Experience at Carnegie Mellon has shown that if a filtering system is made available to users, many will make use of it in order to file messages from specific users or mailing lists. However, many others did not make use of the Andrew system's FLAMES filtering language [FLAMES] due to difficulty in setting it up. Because of the expectation that users will make use of filtering if it is offered and easy to use, this language has been made simple enough to allow many users to make use of it, but rich enough that it can be used productively. However, it is expected that GUI-based editors will be the preferred way of editing filters for a large number of users. 1.1. Conventions Used in This Document In the sections of this document that discuss the requirements of various keywords and operators, the following conventions have been adopted. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [KEYWORDS]. Guenther & Showalter Standards Track [Page 4] RFC 5228 Sieve: An Email Filtering Language January 2008 Each section on a command (test, action, or control) has a line labeled "Usage:". This line describes the usage of the command, including its name and its arguments. Required arguments are listed inside angle brackets ("<" and ">"). Optional arguments are listed inside square brackets ("[" and "]"). Each argument is followed by its type, so "" represents an argument called "key" that is a string. Literal strings are represented with double-quoted strings. Alternatives are separated with slashes, and parentheses are used for grouping, similar to [ABNF]. In the "Usage:" line, there are three special pieces of syntax that are frequently repeated, MATCH-TYPE, COMPARATOR, and ADDRESS-PART. These are discussed in sections 2.7.1, 2.7.3, and 2.7.4, respectively. The formal grammar for these commands is defined in section 8 and is the authoritative reference on how to construct commands, but the formal grammar does not specify the order, semantics, number or types of arguments to commands, or the legal command names. The intent is to allow for extension without changing the grammar. 1.2. Example Mail Messages The following mail messages will be used throughout this document in examples. Message A ----------------------------------------------------------- Date: Tue, 1 Apr 1997 09:06:31 -0800 (PST) From: coyote@desert.example.org To: roadrunner@acme.example.com Subject: I have a present for you Look, I'm sorry about the whole anvil thing, and I really didn't mean to try and drop it on you from the top of the cliff. I want to try to make it up to you. I've got some great birdseed over here at my place--top of the line stuff--and if you come by, I'll have it all wrapped up for you. I'm really sorry for all the problems I've caused for you over the years, but I know we can work this out. -- Wile E. Coyote "Super Genius" coyote@desert.example.org ----------------------------------------------------------- Guenther & Showalter Standards Track [Page 5] RFC 5228 Sieve: An Email Filtering Language January 2008 Message B ----------------------------------------------------------- From: youcouldberich!@reply-by-postal-mail.invalid Sender: b1ff@de.res.example.com To: rube@landru.example.com Date: Mon, 31 Mar 1997 18:26:10 -0800 Subject: $$$ YOU, TOO, CAN BE A MILLIONAIRE! $$$ YOU MAY HAVE ALREADY WON TEN MILLION DOLLARS, BUT I DOUBT IT! SO JUST POST THIS TO SIX HUNDRED NEWSGROUPS! IT WILL GUARANTEE THAT YOU GET AT LEAST FIVE RESPONSES WITH MONEY! MONEY! MONEY! COLD HARD CASH! YOU WILL RECEIVE OVER $20,000 IN LESS THAN TWO MONTHS! AND IT'S LEGAL!!!!!!!!! !!!!!!!!!!!!!!!!!!111111111!!!!!!!11111111111!!1 JUST SEND $5 IN SMALL, UNMARKED BILLS TO THE ADDRESSES BELOW! ----------------------------------------------------------- 2. Design 2.1. Form of the Language The language consists of a set of commands. Each command consists of a set of tokens delimited by whitespace. The command identifier is the first token and it is followed by zero or more argument tokens. Arguments may be literal data, tags, blocks of commands, or test commands. With the exceptions of strings and comments, the language is limited to US-ASCII characters. Strings and comments may contain octets outside the US-ASCII range. Specifically, they will normally be in UTF-8, as specified in [UTF-8]. NUL (US-ASCII 0) is never permitted in scripts, while CR and LF can only appear as the CRLF line ending. Note: While this specification permits arbitrary octets to appear in Sieve scripts inside strings and comments, this has made it difficult to robustly handle Sieve scripts in programs that are sensitive to the encodings used. The "encoded-character" capability (section 2.4.2.4) provides an alternative means of representing such octets in strings using just US-ASCII characters. As such, the use of non-UTF-8 text in scripts should be considered a deprecated feature that may be abandoned. Tokens other than strings are considered case-insensitive. Guenther & Showalter Standards Track [Page 6] RFC 5228 Sieve: An Email Filtering Language January 2008 2.2. Whitespace Whitespace is used to separate tokens. Whitespace is made up of tabs, newlines (CRLF, never just CR or LF), and the space character. The amount of whitespace used is not significant. 2.3. Comments Two types of comments are offered. Comments are semantically equivalent to whitespace and can be used anyplace that whitespace is (with one exception in multi-line strings, as described in the grammar). Hash comments begin with a "#" character that is not contained within a string and continue until the next CRLF. Example: if size :over 100k { # this is a comment discard; } Bracketed comments begin with the token "/*" and end with "*/" outside of a string. Bracketed comments may span multiple lines. Bracketed comments do not nest. Example: if size :over 100K { /* this is a comment this is still a comment */ discard /* this is a comment */ ; } 2.4. Literal Data Literal data means data that is not executed, merely evaluated "as is", to be used as arguments to commands. Literal data is limited to numbers, strings, and string lists. 2.4.1. Numbers Numbers are given as ordinary decimal numbers. As a shorthand for expressing larger values, such as message sizes, a suffix of "K", "M", or "G" MAY be appended to indicate a multiple of a power of two. To be comparable with the power-of-two-based versions of SI units that computers frequently use, "K" specifies kibi-, or 1,024 (2^10) times the value of the number; "M" specifies mebi-, or 1,048,576 (2^20) times the value of the number; and "G" specifies gibi-, or 1,073,741,824 (2^30) times the value of the number [BINARY-SI]. Guenther & Showalter Standards Track [Page 7] RFC 5228 Sieve: An Email Filtering Language January 2008 Implementations MUST support integer values in the inclusive range zero to 2,147,483,647 (2^31 - 1), but MAY support larger values. Only non-negative integers are permitted by this specification. 2.4.2. Strings Scripts involve large numbers of string values as they are used for pattern matching, addresses, textual bodies, etc. Typically, short quoted strings suffice for most uses, but a more convenient form is provided for longer strings such as bodies of messages. A quoted string starts and ends with a single double quote (the <"> character, US-ASCII 34). A backslash ("\", US-ASCII 92) inside of a quoted string is followed by either another backslash or a double quote. These two-character sequences represent a single backslash or double quote within the value, respectively. Scripts SHOULD NOT escape other characters with a backslash. An undefined escape sequence (such as "\a" in a context where "a" has no special meaning) is interpreted as if there were no backslash (in this case, "\a" is just "a"), though that may be changed by extensions. Non-printing characters such as tabs, CRLF, and control characters are permitted in quoted strings. Quoted strings MAY span multiple lines. An unencoded NUL (US-ASCII 0) is not allowed in strings; see section 2.4.2.4 for how it can be encoded. As message header data is converted to [UTF-8] for comparison (see section 2.7.2), most string values will use the UTF-8 encoding. However, implementations MUST accept all strings that match the grammar in section 8. The ability to use non-UTF-8 encoded strings matches existing practice and has proven to be useful both in tests for invalid data and in arguments containing raw MIME parts for extension actions that generate outgoing messages. For entering larger amounts of text, such as an email message, a multi-line form is allowed. It starts with the keyword "text:", followed by a CRLF, and ends with the sequence of a CRLF, a single period, and another CRLF. The CRLF before the final period is considered part of the value. In order to allow the message to contain lines with a single dot, lines are dot-stuffed. That is, when composing a message body, an extra '.' is added before each line that begins with a '.'. When the server interprets the script, these extra dots are removed. Note that a line that begins with a dot followed by a non-dot character is not interpreted as dot-stuffed; Guenther & Showalter Standards Track [Page 8] RFC 5228 Sieve: An Email Filtering Language January 2008 that is, ".foo" is interpreted as ".foo". However, because this is potentially ambiguous, scripts SHOULD be properly dot-stuffed so such lines do not appear. Note that a hashed comment or whitespace may occur in between the "text:" and the CRLF, but not within the string itself. Bracketed comments are not allowed here. 2.4.2.1. String Lists When matching patterns, it is frequently convenient to match against groups of strings instead of single strings. For this reason, a list of strings is allowed in many tests, implying that if the test is true using any one of the strings, then the test is true. For instance, the test 'header :contains ["To", "Cc"] ["me@example.com", "me00@landru.example.com"]' is true if either a To header or Cc header of the input message contains either of the email addresses "me@example.com" or "me00@landru.example.com". Conversely, in any case where a list of strings is appropriate, a single string is allowed without being a member of a list: it is equivalent to a list with a single member. This means that the test 'exists "To"' is equivalent to the test 'exists ["To"]'. 2.4.2.2. Headers Headers are a subset of strings. In the Internet Message Specification [IMAIL], each header line is allowed to have whitespace nearly anywhere in the line, including after the field name and before the subsequent colon. Extra spaces between the header name and the ":" in a header field are ignored. A header name never contains a colon. The "From" header refers to a line beginning "From:" (or "From :", etc.). No header will match the string "From:" due to the trailing colon. Similarly, no header will match a syntactically invalid header name. An implementation MUST NOT cause an error for syntactically invalid header names in tests. Header lines are unfolded as described in [IMAIL] section 2.2.3. Interpretation of header data SHOULD be done according to [MIME3] section 6.2 (see section 2.7.2 below for details). Guenther & Showalter Standards Track [Page 9] RFC 5228 Sieve: An Email Filtering Language January 2008 2.4.2.3. Addresses A number of commands call for email addresses, which are also a subset of strings. When these addresses are used in outbound contexts, addresses must be compliant with [IMAIL], but are further constrained within this document. Using the symbols defined in [IMAIL], section 3, the syntax of an address is: sieve-address = addr-spec ; simple address / phrase "<" addr-spec ">" ; name & addr-spec That is, routes and group syntax are not permitted. If multiple addresses are required, use a string list. Named groups are not permitted. It is an error for a script to execute an action with a value for use as an outbound address that doesn't match the "sieve-address" syntax. 2.4.2.4. Encoding Characters Using "encoded-character" When the "encoded-character" extension is in effect, certain character sequences in strings are replaced by their decoded value. This happens after escape sequences are interpreted and dot- unstuffing has been done. Implementations SHOULD support "encoded- character". Arbitrary octets can be embedded in strings by using the syntax encoded-arb-octets. The sequence is replaced by the octets with the hexadecimal values given by each hex-pair. blank = WSP / CRLF encoded-arb-octets = "${hex:" hex-pair-seq "}" hex-pair-seq = *blank hex-pair *(1*blank hex-pair) *blank hex-pair = 1*2HEXDIG Where WSP and HEXDIG non-terminals are defined in Appendix B.1 of [ABNF]. It may be inconvenient or undesirable to enter Unicode characters verbatim, and for these cases the syntax encoded-unicode-char can be used. The sequence is replaced by the UTF-8 encoding of the specified Unicode characters, which are identified by the hexadecimal value of unicode-hex. encoded-unicode-char = "${unicode:" unicode-hex-seq "}" unicode-hex-seq = *blank unicode-hex *(1*blank unicode-hex) *blank unicode-hex = 1*HEXDIG Guenther & Showalter Standards Track [Page 10] RFC 5228 Sieve: An Email Filtering Language January 2008 It is an error for a script to use a hexadecimal value that isn't in either the range 0 to D7FF or the range E000 to 10FFFF. (The range D800 to DFFF is excluded as those character numbers are only used as part of the UTF-16 encoding form and are not applicable to the UTF-8 encoding that the syntax here represents.) Note: Implementations MUST NOT raise an error for an out-of-range Unicode value unless the sequence containing it is well-formed according to the grammar. The capability string for use with the require command is "encoded- character". In the following script, message B is discarded, since the specified test string is equivalent to "$$$". Example: require "encoded-character"; if header :contains "Subject" "$${hex:24 24}" { discard; } The following examples demonstrate valid and invalid encodings and how they are handled: "$${hex:40}" -> "$@" "${hex: 40 }" -> "@" "${HEX: 40}" -> "@" "${hex:40" -> "${hex:40" "${hex:400}" -> "${hex:400}" "${hex:4${hex:30}}" -> "${hex:40}" "${unicode:40}" -> "@" "${ unicode:40}" -> "${ unicode:40}" "${UNICODE:40}" -> "@" "${UnICoDE:0000040}" -> "@" "${Unicode:40}" -> "@" "${Unicode:Cool}" -> "${Unicode:Cool}" "${unicode:200000}" -> error "${Unicode:DF01} -> error 2.5. Tests Tests are given as arguments to commands in order to control their actions. In this document, tests are given to if/elsif to decide which block of code is run. Guenther & Showalter Standards Track [Page 11] RFC 5228 Sieve: An Email Filtering Language January 2008 2.5.1. Test Lists Some tests ("allof" and "anyof", which implement logical "and" and logical "or", respectively) may require more than a single test as an argument. The test-list syntax element provides a way of grouping tests as a comma-separated list in parentheses. Example: if anyof (not exists ["From", "Date"], header :contains "from" "fool@example.com") { discard; } 2.6. Arguments In order to specify what to do, most commands take arguments. There are three types of arguments: positional, tagged, and optional. It is an error for a script, on a single command, to use conflicting arguments or to use a tagged or optional argument more than once. 2.6.1. Positional Arguments Positional arguments are given to a command that discerns their meaning based on their order. When a command takes positional arguments, all positional arguments must be supplied and must be in the order prescribed. 2.6.2. Tagged Arguments This document provides for tagged arguments in the style of CommonLISP. These are also similar to flags given to commands in most command-line systems. A tagged argument is an argument for a command that begins with ":" followed by a tag naming the argument, such as ":contains". This argument means that zero or more of the next tokens have some particular meaning depending on the argument. These next tokens may be literal data, but they are never blocks. Tagged arguments are similar to positional arguments, except that instead of the meaning being derived from the command, it is derived from the tag. Tagged arguments must appear before positional arguments, but they may appear in any order with other tagged arguments. For simplicity of the specification, this is not expressed in the syntax definitions Guenther & Showalter Standards Track [Page 12] RFC 5228 Sieve: An Email Filtering Language January 2008 with commands, but they still may be reordered arbitrarily provided they appear before positional arguments. Tagged arguments may be mixed with optional arguments. Tagged arguments SHOULD NOT take tagged arguments as arguments. 2.6.3. Optional Arguments Optional arguments are exactly like tagged arguments except that they may be left out, in which case a default value is implied. Because optional arguments tend to result in shorter scripts, they have been used far more than tagged arguments. One particularly noteworthy case is the ":comparator" argument, which allows the user to specify which comparator [COLLATION] will be used to compare two strings, since different languages may impose different orderings on UTF-8 [UTF-8] strings. 2.6.4. Types of Arguments Abstractly, arguments may be literal data, tests, or blocks of commands. In this way, an "if" control structure is merely a command that happens to take a test and a block as arguments and may execute the block of code. However, this abstraction is ambiguous from a parsing standpoint. The grammar in section 8.2 presents a parsable version of this: Arguments are string lists (string-lists), numbers, and tags, which may be followed by a test or a test list (test-list), which may be followed by a block of commands. No more than one test or test list, or more than one block of commands, may be used, and commands that end with a block of commands do not end with semicolons. 2.7. String Comparison When matching one string against another, there are a number of ways of performing the match operation. These are accomplished with three types of matches: an exact match, a substring match, and a wildcard glob-style match. These are described below. In order to provide for matches between character sets and case insensitivity, Sieve uses the comparators defined in the Internet Application Protocol Collation Registry [COLLATION]. Guenther & Showalter Standards Track [Page 13] RFC 5228 Sieve: An Email Filtering Language January 2008 However, when a string represents the name of a header, the comparator is never user-specified. Header comparisons are always done with the "i;ascii-casemap" operator, i.e., case-insensitive comparisons, because this is the way things are defined in the message specification [IMAIL]. 2.7.1. Match Type Commands that perform string comparisons may have an optional match type argument. The three match types in this specification are ":contains", ":is", and ":matches". The ":contains" match type describes a substring match. If the value argument contains the key argument as a substring, the match is true. For instance, the string "frobnitzm" contains "frob" and "nit", but not "fbm". The empty key ("") is contained in all values. The ":is" match type describes an absolute match; if the contents of the first string are absolutely the same as the contents of the second string, they match. Only the string "frobnitzm" is the string "frobnitzm". The empty key ("") only ":is" matches with the empty value. The ":matches" match type specifies a wildcard match using the characters "*" and "?"; the entire value must be matched. "*" matches zero or more characters in the value and "?" matches a single character in the value, where the comparator that is used (see section 2.7.3) defines what a character is. For example, the comparators "i;octet" and "i;ascii-casemap" define a character to be a single octet, so "?" will always match exactly one octet when one of those comparators is in use. In contrast, a Unicode-based comparator would define a character to be any UTF-8 octet sequence encoding one Unicode character and thus "?" may match more than one octet. "?" and "*" may be escaped as "\\?" and "\\*" in strings to match against themselves. The first backslash escapes the second backslash; together, they escape the "*". This is awkward, but it is commonplace in several programming languages that use globs and regular expressions. In order to specify what type of match is supposed to happen, commands that support matching take optional arguments ":matches", ":is", and ":contains". Commands default to using ":is" matching if no match type argument is supplied. Note that these modifiers interact with comparators; in particular, only comparators that support the "substring match" operation are suitable for matching with ":contains" or ":matches". It is an error to use a comparator with ":contains" or ":matches" that is not compatible with it. Guenther & Showalter Standards Track [Page 14] RFC 5228 Sieve: An Email Filtering Language January 2008 It is an error to give more than one of these arguments to a given command. For convenience, the "MATCH-TYPE" syntax element is defined here as follows: Syntax: ":is" / ":contains" / ":matches" 2.7.2. Comparisons across Character Sets Messages may involve a number of character sets. In order for comparisons to work across character sets, implementations SHOULD implement the following behavior: Comparisons are performed on octets. Implementations convert text from header fields in all charsets [MIME3] to Unicode, encoded as UTF-8, as input to the comparator (see section 2.7.3). Implementations MUST be capable of converting US-ASCII, ISO-8859- 1, the US-ASCII subset of ISO-8859-* character sets, and UTF-8. Text that the implementation cannot convert to Unicode for any reason MAY be treated as plain US-ASCII (including any [MIME3] syntax) or processed according to local conventions. An encoded NUL octet (character zero) SHOULD NOT cause early termination of the header content being compared against. If implementations fail to support the above behavior, they MUST conform to the following: No two strings can be considered equal if one contains octets greater than 127. 2.7.3. Comparators In order to allow for language-independent, case-independent matches, the match type may be coupled with a comparator name. The Internet Application Protocol Collation Registry [COLLATION] provides the framework for describing and naming comparators. All implementations MUST support the "i;octet" comparator (simply compares octets) and the "i;ascii-casemap" comparator (which treats uppercase and lowercase characters in the US-ASCII subset of UTF-8 as the same). If left unspecified, the default is "i;ascii-casemap". Some comparators may not be usable with substring matches; that is, they may only work with ":is". It is an error to try to use a comparator with ":matches" or ":contains" that is not compatible with it. Guenther & Showalter Standards Track [Page 15] RFC 5228 Sieve: An Email Filtering Language January 2008 Sieve treats a comparator result of "undefined" the same as a result of "no-match". That is, this base specification does not provide any means to directly detect invalid comparator input. A comparator is specified by the ":comparator" option with commands that support matching. This option is followed by a string providing the name of the comparator to be used. For convenience, the syntax of a comparator is abbreviated to "COMPARATOR", and (repeated in several tests) is as follows: Syntax: ":comparator" So in this example, Example: if header :contains :comparator "i;octet" "Subject" "MAKE MONEY FAST" { discard; } would discard any message with subjects like "You can MAKE MONEY FAST", but not "You can Make Money Fast", since the comparator used is case-sensitive. Comparators other than "i;octet" and "i;ascii-casemap" must be declared with require, as they are extensions. If a comparator declared with require is not known, it is an error, and execution fails. If the comparator is not declared with require, it is also an error, even if the comparator is supported. (See section 2.10.5.) Both ":matches" and ":contains" match types are compatible with the "i;octet" and "i;ascii-casemap" comparators and may be used with them. It is an error to give more than one of these arguments to a given command. 2.7.4. Comparisons against Addresses Addresses are one of the most frequent things represented as strings. These are structured, and being able to compare against the local- part or the domain of an address is useful, so some tests that act exclusively on addresses take an additional optional argument that specifies what the test acts on. These optional arguments are ":localpart", ":domain", and ":all", which act on the local-part (left side), the domain-part (right side), and the whole address. Guenther & Showalter Standards Track [Page 16] RFC 5228 Sieve: An Email Filtering Language January 2008 If an address is not syntactically valid, then it will not be matched by tests specifying ":localpart" or ":domain". The kind of comparison done, such as whether or not the test done is case-insensitive, is specified as a comparator argument to the test. If an optional address-part is omitted, the default is ":all". It is an error to give more than one of these arguments to a given command. For convenience, the "ADDRESS-PART" syntax element is defined here as follows: Syntax: ":localpart" / ":domain" / ":all" 2.8. Blocks Blocks are sets of commands enclosed within curly braces and supplied as the final argument to a command. Such a command is a control structure: when executed it has control over the number of times the commands in the block are executed. With the commands supplied in this memo, there are no loops. The control structures supplied--if, elsif, and else--run a block either once or not at all. 2.9. Commands Sieve scripts are sequences of commands. Commands can take any of the tokens above as arguments, and arguments may be either tagged or positional arguments. Not all commands take all arguments. There are three kinds of commands: test commands, action commands, and control commands. The simplest is an action command. An action command is an identifier followed by zero or more arguments, terminated by a semicolon. Action commands do not take tests or blocks as arguments. The actions referenced in this document are: - keep, to save the message in the default location - fileinto, to save the message in a specific mailbox - redirect, to forward the message to another address - discard, to silently throw away the message Guenther & Showalter Standards Track [Page 17] RFC 5228 Sieve: An Email Filtering Language January 2008 A control command is a command that affects the parsing or the flow of execution of the Sieve script in some way. A control structure is a control command that ends with a block instead of a semicolon. A test command is used as part of a control command. It is used to specify whether or not the block of code given to the control command is executed. 2.10. Evaluation 2.10.1. Action Interaction Some actions cannot be used with other actions because the result would be absurd. These restrictions are noted throughout this memo. Extension actions MUST state how they interact with actions defined in this specification. 2.10.2. Implicit Keep Previous experience with filtering systems suggests that cases tend to be missed in scripts. To prevent errors, Sieve has an "implicit keep". An implicit keep is a keep action (see section 4.3) performed in absence of any action that cancels the implicit keep. An implicit keep is performed if a message is not written to a mailbox, redirected to a new address, or explicitly thrown out. That is, if a fileinto, a keep, a redirect, or a discard is performed, an implicit keep is not. Some actions may be defined to not cancel the implicit keep. These actions may not directly affect the delivery of a message, and are used for their side effects. None of the actions specified in this document meet that criteria, but extension actions may. For instance, with any of the short messages offered above, the following script produces no actions. Example: if size :over 500K { discard; } As a result, the implicit keep is taken. Guenther & Showalter Standards Track [Page 18] RFC 5228 Sieve: An Email Filtering Language January 2008 2.10.3. Message Uniqueness in a Mailbox Implementations SHOULD NOT deliver a message to the same mailbox more than once, even if a script explicitly asks for a message to be written to a mailbox twice. The test for equality of two messages is implementation-defined. If a script asks for a message to be written to a mailbox twice, it MUST NOT be treated as an error. 2.10.4. Limits on Numbers of Actions Site policy MAY limit the number of actions taken and MAY impose restrictions on which actions can be used together. In the event that a script hits a policy limit on the number of actions taken for a particular message, an error occurs. Implementations MUST allow at least one keep or one fileinto. If fileinto is not implemented, implementations MUST allow at least one keep. 2.10.5. Extensions and Optional Features Because of the differing capabilities of many mail systems, several features of this specification are optional. Before any of these extensions can be executed, they must be declared with the "require" action. If an extension is not enabled with "require", implementations MUST treat it as if they did not support it at all. This protects scripts from having their behavior altered by extensions that the script author might not have even been aware of. Implementations MUST NOT execute any Sieve script test or command subsequent to "require" if one of the required extensions is unavailable. Note: The reason for this restriction is that prior experiences with languages such as LISP and Tcl suggest that this is a workable way of noting that a given script uses an extension. Extensions that define actions MUST state how they interact with actions discussed in the base specification. Guenther & Showalter Standards Track [Page 19] RFC 5228 Sieve: An Email Filtering Language January 2008 2.10.6. Errors In any programming language, there are compile-time and run-time errors. Compile-time errors are ones in syntax that are detectable if a syntax check is done. Run-time errors are not detectable until the script is run. This includes transient failures like disk full conditions, but also includes issues like invalid combinations of actions. When an error occurs in a Sieve script, all processing stops. Implementations MAY choose to do a full parse, then evaluate the script, then do all actions. Implementations might even go so far as to ensure that execution is atomic (either all actions are executed or none are executed). Other implementations may choose to parse and run at the same time. Such implementations are simpler, but have issues with partial failure (some actions happen, others don't). Implementations MUST perform syntactic, semantic, and run-time checks on code that is actually executed. Implementations MAY perform those checks or any part of them on code that is not reached during execution. When an error happens, implementations MUST notify the user that an error occurred and which actions (if any) were taken, and do an implicit keep. 2.10.7. Limits on Execution Implementations may limit certain constructs. However, this specification places a lower bound on some of these limits. Implementations MUST support fifteen levels of nested blocks. Implementations MUST support fifteen levels of nested test lists. Guenther & Showalter Standards Track [Page 20] RFC 5228 Sieve: An Email Filtering Language January 2008 3. Control Commands Control structures are needed to allow for multiple and conditional actions. 3.1. Control if There are three pieces to if: "if", "elsif", and "else". Each is actually a separate command in terms of the grammar. However, an elsif or else MUST only follow an if or elsif. An error occurs if these conditions are not met. Usage: if Usage: elsif Usage: else The semantics are similar to those of any of the many other programming languages these control structures appear in. When the interpreter sees an "if", it evaluates the test associated with it. If the test is true, it executes the block associated with it. If the test of the "if" is false, it evaluates the test of the first "elsif" (if any). If the test of "elsif" is true, it runs the elsif's block. An elsif may be followed by an elsif, in which case, the interpreter repeats this process until it runs out of elsifs. When the interpreter runs out of elsifs, there may be an "else" case. If there is, and none of the if or elsif tests were true, the interpreter runs the else's block. This provides a way of performing exactly one of the blocks in the chain. In the following example, both messages A and B are dropped. Example: require "fileinto"; if header :contains "from" "coyote" { discard; } elsif header :contains ["subject"] ["$$$"] { discard; } else { fileinto "INBOX"; } Guenther & Showalter Standards Track [Page 21] RFC 5228 Sieve: An Email Filtering Language January 2008 When the script below is run over message A, it redirects the message to acm@example.com; message B, to postmaster@example.com; any other message is redirected to field@example.com. Example: if header :contains ["From"] ["coyote"] { redirect "acm@example.com"; } elsif header :contains "Subject" "$$$" { redirect "postmaster@example.com"; } else { redirect "field@example.com"; } Note that this definition prohibits the "... else if ..." sequence used by C. This is intentional, because this construct produces a shift-reduce conflict. 3.2. Control require Usage: require The require action notes that a script makes use of a certain extension. Such a declaration is required to use the extension, as discussed in section 2.10.5. Multiple capabilities can be declared with a single require. The require command, if present, MUST be used before anything other than a require can be used. An error occurs if a require appears after a command other than require. Example: require ["fileinto", "reject"]; Example: require "fileinto"; require "vacation"; 3.3. Control stop Usage: stop The "stop" action ends all processing. If the implicit keep has not been cancelled, then it is taken. Guenther & Showalter Standards Track [Page 22] RFC 5228 Sieve: An Email Filtering Language January 2008 4. Action Commands This document supplies four actions that may be taken on a message: keep, fileinto, redirect, and discard. Implementations MUST support the "keep", "discard", and "redirect" actions. Implementations SHOULD support "fileinto". Implementations MAY limit the number of certain actions taken (see section 2.10.4). 4.1. Action fileinto Usage: fileinto The "fileinto" action delivers the message into the specified mailbox. Implementations SHOULD support fileinto, but in some environments this may be impossible. Implementations MAY place restrictions on mailbox names; use of an invalid mailbox name MAY be treated as an error or result in delivery to an implementation- defined mailbox. If the specified mailbox doesn't exist, the implementation MAY treat it as an error, create the mailbox, or deliver the message to an implementation-defined mailbox. If the implementation uses a different encoding scheme than UTF-8 for mailbox names, it SHOULD reencode the mailbox name from UTF-8 to its encoding scheme. For example, the Internet Message Access Protocol [IMAP] uses modified UTF-7, such that a mailbox argument of "odds & ends" would appear in IMAP as "odds &- ends". The capability string for use with the require command is "fileinto". In the following script, message A is filed into mailbox "INBOX.harassment". Example: require "fileinto"; if header :contains ["from"] "coyote" { fileinto "INBOX.harassment"; } 4.2. Action redirect Usage: redirect The "redirect" action is used to send the message to another user at a supplied address, as a mail forwarding feature does. The "redirect" action makes no changes to the message body or existing Guenther & Showalter Standards Track [Page 23] RFC 5228 Sieve: An Email Filtering Language January 2008 headers, but it may add new headers. In particular, existing Received headers MUST be preserved and the count of Received headers in the outgoing message MUST be larger than the same count on the message as received by the implementation. (An implementation that adds a Received header before processing the message does not need to add another when redirecting.) The message is sent back out with the address from the redirect command as an envelope recipient. Implementations MAY combine separate redirects for a given message into a single submission with multiple envelope recipients. (This is not a Mail User Agent (MUA)- style forward, which creates a new message with a different sender and message ID, wrapping the old message in a new one.) The envelope sender address on the outgoing message is chosen by the sieve implementation. It MAY be copied from the message being processed. However, if the message being processed has an empty envelope sender address the outgoing message MUST also have an empty envelope sender address. This last requirement is imposed to prevent loops in the case where a message is redirected to an invalid address when then returns a delivery status notification that also ends up being redirected to the same invalid address. A simple script can be used for redirecting all mail: Example: redirect "bart@example.com"; Implementations MUST take measures to implement loop control, possibly including adding headers to the message or counting Received headers as specified in section 6.2 of [SMTP]. If an implementation detects a loop, it causes an error. Implementations MUST provide means of limiting the number of redirects a Sieve script can perform. See section 10 for more details. Implementations MAY ignore a redirect action silently due to policy reasons. For example, an implementation MAY choose not to redirect to an address that is known to be undeliverable. Any ignored redirect MUST NOT cancel the implicit keep. 4.3. Action keep Usage: keep The "keep" action is whatever action is taken in lieu of all other actions, if no filtering happens at all; generally, this simply means to file the message into the user's main mailbox. This command Guenther & Showalter Standards Track [Page 24] RFC 5228 Sieve: An Email Filtering Language January 2008 provides a way to execute this action without needing to know the name of the user's main mailbox, providing a way to call it without needing to understand the user's setup or the underlying mail system. For instance, in an implementation where the IMAP server is running scripts on behalf of the user at time of delivery, a keep command is equivalent to a fileinto "INBOX". Example: if size :under 1M { keep; } else { discard; } Note that the above script is identical to the one below. Example: if not size :under 1M { discard; } 4.4. Action discard Usage: discard Discard is used to silently throw away the message. It does so by simply canceling the implicit keep. If discard is used with other actions, the other actions still happen. Discard is compatible with all other actions. (For instance, fileinto+discard is equivalent to fileinto.) Discard MUST be silent; that is, it MUST NOT return a non-delivery notification of any kind ([DSN], [MDN], or otherwise). In the following script, any mail from "idiot@example.com" is thrown out. Example: if header :contains ["from"] ["idiot@example.com"] { discard; } While an important part of this language, "discard" has the potential to create serious problems for users: Students who leave themselves logged in to an unattended machine in a public computer lab may find their script changed to just "discard". In order to protect users in this situation (along with similar situations), implementations MAY keep messages destroyed by a script for an indefinite period, and MAY disallow scripts that throw out all mail. Guenther & Showalter Standards Track [Page 25] RFC 5228 Sieve: An Email Filtering Language January 2008 5. Test Commands Tests are used in conditionals to decide which part(s) of the conditional to execute. Implementations MUST support these tests: "address", "allof", "anyof", "exists", "false", "header", "not", "size", and "true". Implementations SHOULD support the "envelope" test. 5.1. Test address Usage: address [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] The "address" test matches Internet addresses in structured headers that contain addresses. It returns true if any header contains any key in the specified part of the address, as modified by the comparator and the match keyword. Whether there are other addresses present in the header doesn't affect this test; this test does not provide any way to determine whether an address is the only address in a header. Like envelope and header, this test returns true if any combination of the header-list and key-list arguments match and returns false otherwise. Internet email addresses [IMAIL] have the somewhat awkward characteristic that the local-part to the left of the at-sign is considered case sensitive, and the domain-part to the right of the at-sign is case insensitive. The "address" command does not deal with this itself, but provides the ADDRESS-PART argument for allowing users to deal with it. The address primitive never acts on the phrase part of an email address or on comments within that address. It also never acts on group names, although it does act on the addresses within the group construct. Implementations MUST restrict the address test to headers that contain addresses, but MUST include at least From, To, Cc, Bcc, Sender, Resent-From, and Resent-To, and it SHOULD include any other header that utilizes an "address-list" structured header body. Example: if address :is :all "from" "tim@example.com" { discard; } Guenther & Showalter Standards Track [Page 26] RFC 5228 Sieve: An Email Filtering Language January 2008 5.2. Test allof Usage: allof The "allof" test performs a logical AND on the tests supplied to it. Example: allof (false, false) => false allof (false, true) => false allof (true, true) => true The allof test takes as its argument a test-list. 5.3. Test anyof Usage: anyof The "anyof" test performs a logical OR on the tests supplied to it. Example: anyof (false, false) => false anyof (false, true) => true anyof (true, true) => true 5.4. Test envelope Usage: envelope [COMPARATOR] [ADDRESS-PART] [MATCH-TYPE] The "envelope" test is true if the specified part of the [SMTP] (or equivalent) envelope matches the specified key. This specification defines the interpretation of the (case insensitive) "from" and "to" envelope-parts. Additional envelope-parts may be defined by other extensions; implementations SHOULD consider unknown envelope parts an error. If one of the envelope-part strings is (case insensitive) "from", then matching occurs against the FROM address used in the SMTP MAIL command. The null reverse-path is matched against as the empty string, regardless of the ADDRESS-PART argument specified. If one of the envelope-part strings is (case insensitive) "to", then matching occurs against the TO address used in the SMTP RCPT command that resulted in this message getting delivered to this user. Note that only the most recent TO is available, and only the one relevant to this user. The envelope-part is a string list and may contain more than one parameter, in which case all of the strings specified in the key-list are matched against all parts given in the envelope-part list. Guenther & Showalter Standards Track [Page 27] RFC 5228 Sieve: An Email Filtering Language January 2008 Like address and header, this test returns true if any combination of the envelope-part list and key-list arguments match and returns false otherwise. All tests against envelopes MUST drop source routes. If the SMTP transaction involved several RCPT commands, only the data from the RCPT command that caused delivery to this user is available in the "to" part of the envelope. If a protocol other than SMTP is used for message transport, implementations are expected to adapt this command appropriately. The envelope command is optional. Implementations SHOULD support it, but the necessary information may not be available in all cases. The capability string for use with the require command is "envelope". Example: require "envelope"; if envelope :all :is "from" "tim@example.com" { discard; } 5.5. Test exists Usage: exists The "exists" test is true if the headers listed in the header-names argument exist within the message. All of the headers must exist or the test is false. The following example throws out mail that doesn't have a From header and a Date header. Example: if not exists ["From","Date"] { discard; } 5.6. Test false Usage: false The "false" test always evaluates to false. Guenther & Showalter Standards Track [Page 28] RFC 5228 Sieve: An Email Filtering Language January 2008 5.7. Test header Usage: header [COMPARATOR] [MATCH-TYPE] The "header" test evaluates to true if the value of any of the named headers, ignoring leading and trailing whitespace, matches any key. The type of match is specified by the optional match argument, which defaults to ":is" if not specified, as specified in section 2.6. Like address and envelope, this test returns true if any combination of the header-names list and key-list arguments match and returns false otherwise. If a header listed in the header-names argument exists, it contains the empty key (""). However, if the named header is not present, it does not match any key, including the empty key. So if a message contained the header X-Caffeine: C8H10N4O2 these tests on that header evaluate as follows: header :is ["X-Caffeine"] [""] => false header :contains ["X-Caffeine"] [""] => true Testing whether a given header is either absent or doesn't contain any non-whitespace characters can be done using a negated "header" test: not header :matches "Cc" "?*" 5.8. Test not Usage: not The "not" test takes some other test as an argument, and yields the opposite result. "not false" evaluates to "true" and "not true" evaluates to "false". 5.9. Test size Usage: size <":over" / ":under"> The "size" test deals with the size of a message. It takes either a tagged argument of ":over" or ":under", followed by a number representing the size of the message. Guenther & Showalter Standards Track [Page 29] RFC 5228 Sieve: An Email Filtering Language January 2008 If the argument is ":over", and the size of the message is greater than the number provided, the test is true; otherwise, it is false. If the argument is ":under", and the size of the message is less than the number provided, the test is true; otherwise, it is false. Exactly one of ":over" or ":under" must be specified, and anything else is an error. The size of a message is defined to be the number of octets in the [IMAIL] representation of the message. Note that for a message that is exactly 4,000 octets, the message is neither ":over" nor ":under" 4000 octets. 5.10. Test true Usage: true The "true" test always evaluates to true. 6. Extensibility New control commands, actions, and tests can be added to the language. Sites must make these features known to their users; this document does not define a way to discover the list of extensions supported by the server. Any extensions to this language MUST define a capability string that uniquely identifies that extension. Capability string are case- sensitive; for example, "foo" and "FOO" are different capabilities. If a new version of an extension changes the functionality of a previously defined extension, it MUST use a different name. Extensions may register a set of related capabilities by registering just a unique prefix for them. The "comparator-" prefix is an example of this. The prefix MUST end with a "-" and MUST NOT overlap any existing registrations. In a situation where there is a script submission protocol and an extension advertisement mechanism aware of the details of this language, scripts submitted can be checked against the mail server to prevent use of an extension that the server does not support. Extensions MUST state how they interact with constraints defined in section 2.10, e.g., whether they cancel the implicit keep, and which actions they are compatible and incompatible with. Extensions MUST NOT change the behavior of the "require" control command or alter the interpretation of the argument to the "require" control. Guenther & Showalter Standards Track [Page 30] RFC 5228 Sieve: An Email Filtering Language January 2008 Extensions that can submit new email messages or otherwise generate new protocol requests MUST consider loop suppression, at least to document any security considerations. 6.1. Capability String Capability strings are typically short strings describing what capabilities are supported by the server. Capability strings beginning with "vnd." represent vendor-defined extensions. Such extensions are not defined by Internet standards or RFCs, but are still registered with IANA in order to prevent conflicts. Extensions starting with "vnd." SHOULD be followed by the name of the vendor and product, such as "vnd.acme.rocket-sled". The following capability strings are defined by this document: encoded-character The string "encoded-character" indicates that the implementation supports the interpretation of "${hex:...}" and "${unicode:...}" in strings. envelope The string "envelope" indicates that the implementation supports the "envelope" command. fileinto The string "fileinto" indicates that the implementation supports the "fileinto" command. comparator- The string "comparator-elbonia" is provided if the implementation supports the "elbonia" comparator. Therefore, all implementations have at least the "comparator-i;octet" and "comparator-i;ascii-casemap" capabilities. However, these comparators may be used without being declared with require. 6.2. IANA Considerations In order to provide a standard set of extensions, a registry is maintained by IANA. This registry contains both vendor-controlled capability names (beginning with "vnd.") and IETF-controlled capability names. Vendor-controlled capability names may be registered on a first-come, first-served basis, by applying to IANA with the form in the following section. Registration of capability prefixes that do not begin with "vnd." REQUIRES a standards track or IESG-approved experimental RFC. Extensions designed for interoperable use SHOULD use IETF-controlled capability names. Guenther & Showalter Standards Track [Page 31] RFC 5228 Sieve: An Email Filtering Language January 2008 6.2.1. Template for Capability Registrations The following template is to be used for registering new Sieve extensions with IANA. To: iana@iana.org Subject: Registration of new Sieve extension Capability name: [the string for use in the 'require' statement] Description: [a brief description of what the extension adds or changes] RFC number: [for extensions published as RFCs] Contact address: [email and/or physical address to contact for additional information] 6.2.2. Handling of Existing Capability Registrations In order to bring the existing capability registrations in line with the new template, IANA has modified each as follows: 1. The "capability name" and "capability arguments" fields have been eliminated 2. The "capability keyword" field have been renamed to "Capability name" 3. An empty "Description" field has been added 4. The "Standards Track/IESG-approved experimental RFC number" field has been renamed to "RFC number" 5. The "Person and email address to contact for further information" field should be renamed to "Contact address" 6.2.3. Initial Capability Registrations This RFC updates the following entries in the IANA registry for Sieve extensions. Capability name: encoded-character Description: changes the interpretation of strings to allow arbitrary octets and Unicode characters to be represented using US-ASCII RFC number: RFC 5228 (Sieve base spec) Contact address: The Sieve discussion list Capability name: fileinto Description: adds the 'fileinto' action for delivering to a mailbox other than the default RFC number: RFC 5228 (Sieve base spec) Contact address: The Sieve discussion list Guenther & Showalter Standards Track [Page 32] RFC 5228 Sieve: An Email Filtering Language January 2008 Capability name: envelope Description: adds the 'envelope' test for testing the message transport sender and recipient address RFC number: RFC 5228 (Sieve base spec) Contact address: The Sieve discussion list Capability name: comparator-* (anything starting with "comparator-") Description: adds the indicated comparator for use with the :comparator argument RFC number: RFC 5228 (Sieve base spec) and [COLLATION] Contact address: The Sieve discussion list 6.3. Capability Transport A method of advertising which capabilities an implementation supports is difficult due to the wide range of possible implementations. Such a mechanism, however, should have the property that the implementation can advertise the complete set of extensions that it supports. 7. Transmission The [MIME] type for a Sieve script is "application/sieve". The registration of this type for RFC 2048 requirements is updated as follows: Subject: Registration of MIME media type application/sieve MIME media type name: application MIME subtype name: sieve Required parameters: none Optional parameters: none Encoding considerations: Most Sieve scripts will be textual, written in UTF-8. When non-7bit characters are used, quoted-printable is appropriate for transport systems that require 7bit encoding. Security considerations: Discussed in section 10 of this RFC. Interoperability considerations: Discussed in section 2.10.5 of this RFC. Published specification: this RFC. Applications that use this media type: sieve-enabled mail servers and clients Additional information: Magic number(s): File extension(s): .siv .sieve Macintosh File Type Code(s): Guenther & Showalter Standards Track [Page 33] RFC 5228 Sieve: An Email Filtering Language January 2008 Person & email address to contact for further information: See the discussion list at ietf-mta-filters@imc.org. Intended usage: COMMON Author/Change controller: The SIEVE WG, delegated by the IESG. 8. Parsing The Sieve grammar is separated into tokens and a separate grammar as most programming languages are. Additional rules are supplied here for common arguments to various language facilities. 8.1. Lexical Tokens Sieve scripts are encoded in UTF-8. The following assumes a valid UTF-8 encoding; special characters in Sieve scripts are all US-ASCII. The following are tokens in Sieve: - identifiers - tags - numbers - quoted strings - multi-line strings - other separators Identifiers, tags, and numbers are case-insensitive, while quoted strings and multi-line strings are case-sensitive. Blanks, horizontal tabs, CRLFs, and comments ("whitespace") are ignored except as they separate tokens. Some whitespace is required to separate otherwise adjacent tokens and in specific places in the multi-line strings. CR and LF can only appear in CRLF pairs. The other separators are single individual characters and are mentioned explicitly in the grammar. The lexical structure of sieve is defined in the following grammar (as described in [ABNF]): bracket-comment = "/*" *not-star 1*STAR *(not-star-slash *not-star 1*STAR) "/" ; No */ allowed inside a comment. ; (No * is allowed unless it is the last ; character, or unless it is followed by a ; character that isn't a slash.) Guenther & Showalter Standards Track [Page 34] RFC 5228 Sieve: An Email Filtering Language January 2008 comment = bracket-comment / hash-comment hash-comment = "#" *octet-not-crlf CRLF identifier = (ALPHA / "_") *(ALPHA / DIGIT / "_") multi-line = "text:" *(SP / HTAB) (hash-comment / CRLF) *(multiline-literal / multiline-dotstart) "." CRLF multiline-literal = [ octet-not-period *octet-not-crlf ] CRLF multiline-dotstart = "." 1*octet-not-crlf CRLF ; A line containing only "." ends the ; multi-line. Remove a leading '.' if ; followed by another '.'. not-star = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-FF ; either a CRLF pair, OR a single octet ; other than NUL, CR, LF, or star not-star-slash = CRLF / %x01-09 / %x0B-0C / %x0E-29 / %x2B-2E / %x30-FF ; either a CRLF pair, OR a single octet ; other than NUL, CR, LF, star, or slash number = 1*DIGIT [ QUANTIFIER ] octet-not-crlf = %x01-09 / %x0B-0C / %x0E-FF ; a single octet other than NUL, CR, or LF octet-not-period = %x01-09 / %x0B-0C / %x0E-2D / %x2F-FF ; a single octet other than NUL, ; CR, LF, or period octet-not-qspecial = %x01-09 / %x0B-0C / %x0E-21 / %x23-5B / %x5D-FF ; a single octet other than NUL, ; CR, LF, double-quote, or backslash QUANTIFIER = "K" / "M" / "G" quoted-other = "\" octet-not-qspecial ; represents just the octet-no-qspecial ; character. SHOULD NOT be used quoted-safe = CRLF / octet-not-qspecial ; either a CRLF pair, OR a single octet other ; than NUL, CR, LF, double-quote, or backslash Guenther & Showalter Standards Track [Page 35] RFC 5228 Sieve: An Email Filtering Language January 2008 quoted-special = "\" (DQUOTE / "\") ; represents just a double-quote or backslash quoted-string = DQUOTE quoted-text DQUOTE quoted-text = *(quoted-safe / quoted-special / quoted-other) STAR = "*" tag = ":" identifier white-space = 1*(SP / CRLF / HTAB) / comment 8.2. Grammar The following is the grammar of Sieve after it has been lexically interpreted. No whitespace or comments appear below. The start symbol is "start". argument = string-list / number / tag arguments = *argument [ test / test-list ] block = "{" commands "}" command = identifier arguments (";" / block) commands = *command start = commands string = quoted-string / multi-line string-list = "[" string *("," string) "]" / string ; if there is only a single string, the brackets ; are optional test = identifier arguments test-list = "(" test *("," test) ")" 8.3. Statement Elements These elements are collected from the "Syntax" sections elsewhere in this document, and are provided here in [ABNF] syntax so that they can be modified by extensions. ADDRESS-PART = ":localpart" / ":domain" / ":all" Guenther & Showalter Standards Track [Page 36] RFC 5228 Sieve: An Email Filtering Language January 2008 COMPARATOR = ":comparator" string MATCH-TYPE = ":is" / ":contains" / ":matches" 9. Extended Example The following is an extended example of a Sieve script. Note that it does not make use of the implicit keep. # # Example Sieve Filter # Declare any optional features or extension used by the script # require ["fileinto"]; # # Handle messages from known mailing lists # Move messages from IETF filter discussion list to filter mailbox # if header :is "Sender" "owner-ietf-mta-filters@imc.org" { fileinto "filter"; # move to "filter" mailbox } # # Keep all messages to or from people in my company # elsif address :DOMAIN :is ["From", "To"] "example.com" { keep; # keep in "In" mailbox } # # Try and catch unsolicited email. If a message is not to me, # or it contains a subject known to be spam, file it away. # elsif anyof (NOT address :all :contains ["To", "Cc", "Bcc"] "me@example.com", header :matches "subject" ["*make*money*fast*", "*university*dipl*mas*"]) { fileinto "spam"; # move to "spam" mailbox } else { # Move all other (non-company) mail to "personal" # mailbox. fileinto "personal"; } Guenther & Showalter Standards Track [Page 37] RFC 5228 Sieve: An Email Filtering Language January 2008 10. Security Considerations Users must get their mail. It is imperative that whatever implementations use to store the user-defined filtering scripts protect them from unauthorized modification, to preserve the integrity of the mail system. An attacker who can modify a script can cause mail to be discarded, rejected, or forwarded to an unauthorized recipient. In addition, it's possible that Sieve scripts might expose private information, such as mailbox names, or email addresses of favored (or disfavored) correspondents. Because of that, scripts SHOULD also be protected from unauthorized retrieval. Several commands, such as "discard", "redirect", and "fileinto", allow for actions to be taken that are potentially very dangerous. Use of the "redirect" command to generate notifications may easily overwhelm the target address, especially if it was not designed to handle large messages. Allowing a single script to redirect to multiple destinations can be used as a means of amplifying the number of messages in an attack. Moreover, if loop detection is not properly implemented, it may be possible to set up exponentially growing message loops. Accordingly, Sieve implementations: (1) MUST implement facilities to detect and break message loops. See section 6.2 of [SMTP] for additional information on basic loop detection strategies. (2) MUST provide the means for administrators to limit the ability of users to abuse redirect. In particular, it MUST be possible to limit the number of redirects a script can perform. Additionally, if no use cases exist for using redirect to multiple destinations, this limit SHOULD be set to 1. Additional limits, such as the ability to restrict redirect to local users, MAY also be implemented. (3) MUST provide facilities to log use of redirect in order to facilitate tracking down abuse. (4) MAY use script analysis to determine whether or not a given script can be executed safely. While the Sieve language is sufficiently complex that full analysis of all possible scripts is computationally infeasible, the majority of real-world scripts are amenable to analysis. For example, an implementation might Guenther & Showalter Standards Track [Page 38] RFC 5228 Sieve: An Email Filtering Language January 2008 allow scripts that it has determined are safe to run unhindered, block scripts that are potentially problematic, and subject unclassifiable scripts to additional auditing and logging. Allowing redirects at all may not be appropriate in situations where email accounts are freely available and/or not trackable to a human who can be held accountable for creating message bombs or other abuse. As with any filter on a message stream, if the Sieve implementation and the mail agents 'behind' Sieve in the message stream differ in their interpretation of the messages, it may be possible for an attacker to subvert the filter. Of particular note are differences in the interpretation of malformed messages (e.g., missing or extra syntax characters) or those that exhibit corner cases (e.g., NUL octets encoded via [MIME3]). 11. Acknowledgments This document has been revised in part based on comments and discussions that took place on and off the SIEVE mailing list. Thanks to Sharon Chisholm, Cyrus Daboo, Ned Freed, Arnt Gulbrandsen, Michael Haardt, Kjetil Torgrim Homme, Barry Leiba, Mark E. Mallett, Alexey Melnikov, Eric Rescorla, Rob Siemborski, and Nigel Swinson for reviews and suggestions. 12. Normative References [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", RFC 4234, October 2005. [COLLATION] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet Application Protocol Collation Registry", RFC 4790, March 2007. [IMAIL] Resnick, P., Ed., "Internet Message Format", RFC 2822, April 2001. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [MIME] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [MIME3] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. Guenther & Showalter Standards Track [Page 39] RFC 5228 Sieve: An Email Filtering Language January 2008 [SMTP] Klensin, J., Ed., "Simple Mail Transfer Protocol", RFC 2821, April 2001. [UTF-8] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. 13. Informative References [BINARY-SI] "Standard IEC 60027-2: Letter symbols to be used in electrical technology - Part 2: Telecommunications and electronics", January 1999. [DSN] Moore, K. and G. Vaudreuil, "An Extensible Message Format for Delivery Status Notifications", RFC 3464, January 2003. [FLAMES] Borenstein, N, and C. Thyberg, "Power, Ease of Use, and Cooperative Work in a Practical Multimedia Message System", Int. J. of Man-Machine Studies, April, 1991. Reprinted in Computer-Supported Cooperative Work and Groupware, Saul Greenberg, editor, Harcourt Brace Jovanovich, 1991. Reprinted in Readings in Groupware and Computer-Supported Cooperative Work, Ronald Baecker, editor, Morgan Kaufmann, 1993. [IMAP] Crispin, M., "Internet Message Access Protocol - version 4rev1", RFC 3501, March 2003. [MDN] Hansen, T., Ed., and G. Vaudreuil, Ed., "Message Disposition Notification", RFC 3798, May 2004. [RFC3028] Showalter, T., "Sieve: A Mail Filtering Language", RFC 3028, January 2001. Guenther & Showalter Standards Track [Page 40] RFC 5228 Sieve: An Email Filtering Language January 2008 14. Changes from RFC 3028 This following list is a summary of the changes that have been made in the Sieve language base specification from [RFC3028]. 1. Removed ban on tests having side-effects 2. Removed reject extension (will be specified in a separate RFC) 3. Clarified description of comparators to match [COLLATION], the new base specification for them 4. Require stripping of leading and trailing whitespace in "header" test 5. Clarified or tightened handling of many minor items, including: - invalid [MIME3] encoding - invalid addresses in headers - invalid header field names in tests - 'undefined' comparator result - unknown envelope parts - null return-path in "envelope" test 6. Capability strings are case-sensitive 7. Clarified that fileinto should reencode non-ASCII mailbox names to match the mailstore's conventions 8. Errors in the ABNF were corrected 9. The references were updated and split into normative and informative 10. Added encoded-character capability and deprecated (but did not remove) use of arbitrary binary octets in Sieve scripts. 11. Updated IANA registration template, and added IANA considerations to permit capability prefix registrations. 12. Added .sieve as a valid extension for Sieve scripts. Editors' Addresses Philip Guenther Sendmail, Inc. 6425 Christie St. Ste 400 Emeryville, CA 94608 EMail: guenther@sendmail.com Tim Showalter EMail: tjs@psaux.com Guenther & Showalter Standards Track [Page 41] RFC 5228 Sieve: An Email Filtering Language January 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Guenther & Showalter Standards Track [Page 42] ================================================ FILE: docs/rfcs/rfc5255.IMAP_i18n.txt ================================================ Network Working Group C. Newman Request for Comments: 5255 Sun Microsystems Category: Standards Track A. Gulbrandsen Oryx Mail Systems GmhH A. Melnikov Isode Limited June 2008 Internet Message Access Protocol Internationalization Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract Internet Message Access Protocol (IMAP) version 4rev1 has basic support for non-ASCII characters in mailbox names and search substrings. It also supports non-ASCII message headers and content encoded as specified by Multipurpose Internet Mail Extensions (MIME). This specification defines a collection of IMAP extensions that improve international support including language negotiation for international error text, translations for namespace prefixes, and comparator negotiation for search, sort, and thread. Newman, et al. Standards Track [Page 1] RFC 5255 IMAP Internationalization June 2008 Table of Contents 1. Introduction ....................................................3 2. Conventions Used in This Document ...............................3 3. LANGUAGE Extension ..............................................3 3.1. LANGUAGE Extension Requirements ............................4 3.2. LANGUAGE Command ...........................................4 3.3. LANGUAGE Response ..........................................6 3.4. TRANSLATION Extension to the NAMESPACE Response ............7 3.5. Formal Syntax ..............................................8 4. I18NLEVEL=1 and I18NLEVEL=2 Extensions ..........................9 4.1. Introduction and Overview ..................................9 4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 ....9 4.3. I18NLEVEL=1 Extension Requirements ........................10 4.4. I18NLEVEL=2 Extension Requirements ........................10 4.5. Compatibility Notes .......................................11 4.6. Comparators and Character Encodings .......................11 4.7. COMPARATOR Command ........................................13 4.8. COMPARATOR Response .......................................14 4.9. BADCOMPARATOR Response Code ...............................14 4.10. Formal Syntax ............................................14 5. Other IMAP Internationalization Issues .........................15 5.1. Unicode Userids and Passwords .............................15 5.2. UTF-8 Mailbox Names .......................................15 5.3. UTF-8 Domains, Addresses, and Mail Headers ................15 6. IANA Considerations ............................................16 7. Security Considerations ........................................16 8. Acknowledgements ...............................................16 9. Relevant Sources of Documents for Internationalized IMAP Implementations ................................................17 10. Normative References ..........................................17 11. Informative References ........................................18 Newman, et al. Standards Track [Page 2] RFC 5255 IMAP Internationalization June 2008 1. Introduction This specification defines two IMAP4rev1 [RFC3501] extensions to enhance international support. These extensions can be advertised and implemented separately. The LANGUAGE extension allows the client to request a suitable language for protocol error messages and in combination with the NAMESPACE extension [RFC2342] enables namespace translations. The I18NLEVEL=2 extension allows the client to request a suitable collation that will modify the behavior of the base specification's SEARCH command as well as the SORT and THREAD extensions [SORT]. This leverages the collation registry [RFC4790]. The I18NLEVEL=1 extension updates SEARCH/SORT/THREAD to use i;unicode-casemap comparator, as defined in [UCM]. I18NLEVEL=1 is a simpler version of I18NLEVEL=2 with no ability to select a different collation. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. The formal syntax uses the Augmented Backus-Naur Form (ABNF) [RFC5234] notation including the core rules defined in Appendix A. The UTF-8-related productions are defined in [RFC3629]. In examples, "C:" and "S:" indicate lines sent by the client and server respectively. If a single "C:" or "S:" label applies to multiple lines, then the line breaks between those lines are for editorial clarity only and are not part of the actual protocol exchange. 3. LANGUAGE Extension IMAP allows server responses to include human-readable text that in many cases needs to be presented to the user. But that text is limited to US-ASCII by the IMAP specification [RFC3501] in order to preserve backwards compatibility with deployed IMAP implementations. This section specifies a way for an IMAP client to negotiate which language the server should use when sending human-readable text. Newman, et al. Standards Track [Page 3] RFC 5255 IMAP Internationalization June 2008 The LANGUAGE extension only provides a mechanism for altering fixed server strings such as response text and NAMESPACE folder names. Assigning localized language aliases to shared mailboxes would be done with a separate mechanism such as the proposed METADATA extension (see [METADATA]). 3.1. LANGUAGE Extension Requirements IMAP servers that support this extension MUST list the keyword LANGUAGE in their CAPABILITY response as well as in the greeting CAPABILITY data. A server that advertises this extension MUST use the language "i-default" as described in [RFC2277] as its default language until another supported language is negotiated by the client. A server MUST include "i-default" as one of its supported languages. IMAP servers SHOULD NOT advertise the LANGUAGE extension if they discover that they only support "i-default". Clients and servers that support this extension MUST also support the NAMESPACE extension [RFC2342]. The LANGUAGE command is valid in all states. Clients SHOULD issue LANGUAGE before authentication, since some servers send valuable user information as part of authentication (e.g., "password is correct, but expired"). If a security layer (such as SASL or TLS) is subsequently negotiated by the client, it MUST re-issue the LANGUAGE command in order to make sure that no previous active attack (if any) on LANGUAGE negotiation has effect on subsequent error messages. (See Section 7 for a more detailed explanation of the attack.) 3.2. LANGUAGE Command Arguments: Optional language range arguments. Response: A possible LANGUAGE response (see Section 3.3). A possible NAMESPACE response (see Section 3.4). Result: OK - Command completed NO - Could not complete command BAD - Arguments invalid The LANGUAGE command requests that human-readable text emitted by the server be localized to a language matching one of the language range argument as described by Section 2 of [RFC4647]. Newman, et al. Standards Track [Page 4] RFC 5255 IMAP Internationalization June 2008 If the command succeeds, the server will return human-readable responses in the first supported language specified. These responses will be in UTF-8 [RFC3629]. The server MUST send a LANGUAGE response specifying the language used, and the change takes effect immediately after the LANGUAGE response. If the command fails, the server continues to return human-readable responses in the language it was previously using. The special "default" language range argument indicates a request to use a language designated as preferred by the server administrator. The preferred language MAY vary based on the currently active user. If a language range does not match a known language tag exactly but does match a language by the rules of [RFC4647], the server MUST send an untagged LANGUAGE response indicating the language selected. If there aren't any arguments, the server SHOULD send an untagged LANGUAGE response listing the languages it supports. If the server is unable to enumerate the list of languages it supports it MAY return a tagged NO response to the enumeration request. If, after receiving a LANGUAGE request, the server discovers that it doesn't support any language other than i-default, it MUST return a tagged NO response to the enumeration request. < The server defaults to using English i-default responses until the user explicitly changes the language. > C: A001 LOGIN KAREN PASSWORD S: A001 OK LOGIN completed < Client requested MUL language, which no server supports. > C: A002 LANGUAGE MUL S: A002 NO Unsupported language MUL < A LANGUAGE command with no arguments is a request to enumerate the list of languages the server supports. > C: A003 LANGUAGE S: * LANGUAGE (EN DE IT i-default) S: A003 OK Supported languages have been enumerated C: B001 LANGUAGE S: B001 NO Server is unable to enumerate supported languages Newman, et al. Standards Track [Page 5] RFC 5255 IMAP Internationalization June 2008 < Once the client changes the language, all responses will be in that language starting after the LANGUAGE response. Note that this includes the NAMESPACE response. Because RFCs are in US- ASCII, this document uses an ASCII transcription rather than UTF-8 text, e.g., "ue" in the word "ausgefuehrt" > C: C001 LANGUAGE DE S: * LANGUAGE (DE) S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) S: C001 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt < If a server does not support the requested primary language, responses will continue to be returned in the current language the server is using. > C: D001 LANGUAGE FR S: D001 NO Diese Sprache ist nicht unterstuetzt C: D002 LANGUAGE DE-IT S: * LANGUAGE (DE-IT) S: * NAMESPACE (("" "/"))(("Other Users/" "/" "TRANSLATION" ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) S: D002 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt C: D003 LANGUAGE "default" S: * LANGUAGE (DE) S: D003 OK Sprachwechsel durch LANGUAGE-Befehl ausgefuehrt < Server does not speak French, but does speak English. User speaks Canadian French and Canadian English. > C: E001 LANGUAGE FR-CA EN-CA S: * LANGUAGE (EN) S: E001 OK Now speaking English 3.3. LANGUAGE Response Contents: A list of one or more language tags. The LANGUAGE response occurs as a result of a LANGUAGE command. A LANGUAGE response with a list containing a single language tag indicates that the server is now using that language. A LANGUAGE response with a list containing multiple language tags indicates the server is communicating a list of available languages to the client, and no change in the active language has been made. Newman, et al. Standards Track [Page 6] RFC 5255 IMAP Internationalization June 2008 3.4. TRANSLATION Extension to the NAMESPACE Response If localized representations of the namespace prefixes are available in the selected language, the server SHOULD include these in the TRANSLATION extension to the NAMESPACE response. The TRANSLATION extension to the NAMESPACE response returns a single string, containing the modified UTF-7 [RFC3501] encoded translation of the namespace prefix. It is the responsibility of the client to convert between the namespace prefix and the translation of the namespace prefix when presenting mailbox names to the user. In this example, a server supports the IMAP4 NAMESPACE command. It uses no prefix to the user's Personal Namespace, a prefix of "Other Users" to its Other Users' Namespace, and a prefix of "Public Folders" to its only Shared Namespace. Since a client will often display these prefixes to the user, the server includes a translation of them that can be presented to the user. C: A001 LANGUAGE DE-IT S: * NAMESPACE (("" "/")) (("Other Users/" "/" "TRANSLATION" ("Andere Ben&APw-tzer/"))) (("Public Folders/" "/" "TRANSLATION" ("Gemeinsame Postf&AM8-cher/"))) S: A001 OK LANGUAGE-Befehl ausgefuehrt Newman, et al. Standards Track [Page 7] RFC 5255 IMAP Internationalization June 2008 3.5. Formal Syntax The following syntax specification inherits ABNF [RFC5234] rules from IMAP4rev1 [RFC3501], IMAP4 Namespace [RFC2342], Tags for the Identifying Languages [RFC4646], UTF-8 [RFC3629], and Collected Extensions to IMAP4 ABNF [RFC4466]. command-any =/ language-cmd ; LANGUAGE command is valid in all states language-cmd = "LANGUAGE" *(SP lang-range-quoted) response-payload =/ language-data language-data = "LANGUAGE" SP "(" lang-tag-quoted *(SP lang-tag-quoted) ")" namespace-trans = SP DQUOTE "TRANSLATION" DQUOTE SP "(" string ")" ; the string is encoded in Modified UTF-7. ; this is a subset of the syntax permitted by ; the Namespace-Response-Extension rule in [RFC4466] lang-range-quoted = astring ; Once any literal wrapper or quoting is removed, this ; follows the language-range rule in [RFC4647] lang-tag-quoted = astring ; Once any literal wrapper or quoting is removed, this follows ; the Language-Tag rule in [RFC4646] resp-text = ["[" resp-text-code "]" SP ] UTF8-TEXT-CHAR *(UTF8-TEXT-CHAR / "[") ; After the server is changed to a language other than ; i-default, this resp-text rule replaces the resp-text ; rule from [RFC3501]. UTF8-TEXT-CHAR = %x20-5A / %x5C-7E / UTF8-2 / UTF8-3 / UTF8-4 ; UTF-8 excluding 7-bit control characters and "[" Newman, et al. Standards Track [Page 8] RFC 5255 IMAP Internationalization June 2008 4. I18NLEVEL=1 and I18NLEVEL=2 Extensions 4.1. Introduction and Overview IMAP4rev1 [RFC3501] includes the SEARCH command that can be used to locate messages matching criteria including human-readable text. The SORT extension [SORT] to IMAP allows the client to ask the server to determine the order of messages based on criteria including human- readable text. These mechanisms require the ability to support non- English search and sort functions. Section 4 defines two IMAP extensions for internationalizing IMAP SEARCH, SORT, and THREAD [SORT] using the comparator framework [RFC4790]. The I18NLEVEL=1 extension updates SEARCH/SORT/THREAD to use i;unicode-casemap comparator, as defined in [UCM]. See Sections 4.2 and 4.3 for more details. The I18NLEVEL=2 extension is a superset of the I18NLEVEL=1 extension. It adds to I18NLEVEL=1 extension the ability to determine the active comparator (see definition below) and to negotiate use of comparators using the COMPARATOR command. It also adds the COMPARATOR response that indicates the active comparator and possibly other available comparators. See Sections 4.2 and 4.4 for more details. 4.2. Requirements Common to Both I18NLEVEL=1 and I18NLEVEL=2 The term "default comparator" refers to the comparator that is used by SEARCH and SORT absent any negotiation using the COMPARATOR command (see Section 4.7). The term "active comparator" refers to the comparator which will be used within a session, e.g., by SEARCH and SORT. The COMPARATOR command is used to change the active comparator. The active comparator applies to the following SEARCH keys: "BCC", "BODY", "CC", "FROM", "SUBJECT", "TEXT", "TO", and "HEADER". If the server also advertises the "SORT" extension, then the active comparator applies to the following SORT keys: "CC", "FROM", "SUBJECT", and "TO". If the server advertises THREAD=ORDEREDSUBJECT, then the active comparator applies to the ORDEREDSUBJECT threading algorithm. If the server advertises THREAD=REFERENCES, then the active comparator applies to the subject field comparisons done by REFERENCES threading algorithm. Future extensions may choose to apply the active comparator to their SEARCH keys. Newman, et al. Standards Track [Page 9] RFC 5255 IMAP Internationalization June 2008 For SORT and THREAD, the pre-processing necessary to extract the base subject text from a Subject header occurs prior to the application of a comparator. A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST implement the i;unicode-casemap comparator, as defined in [UCM]. A server that advertises I18NLEVEL=1 or I18NLEVEL=2 extension MUST support UTF-8 as a SEARCH charset. 4.3. I18NLEVEL=1 Extension Requirements An IMAP server that satisfies all requirements specified in Sections 4.2 and 4.6 (and that doesn't support/advertise any other I18NLEVEL= extension, where n > 1) MUST list the keyword I18NLEVEL=1 in its CAPABILITY data once IMAP enters the authenticated state, and MAY list that keyword in other states. 4.4. I18NLEVEL=2 Extension Requirements An IMAP server that satisfies all requirements specified in Sections 4.2, 4.4, and 4.6-4.10 (and that doesn't support/advertise any other I18NLEVEL= extension, where n > 2) MUST list the keyword I18NLEVEL=2 in its CAPABILITY data once IMAP enters the authenticated state, and MAY list that keyword in other states. A server that advertises this extension MUST implement the i;unicode-casemap comparator, as defined in [UCM]. It MAY implement other comparators from the IANA registry established by [RFC4790]. See also Section 4.5 of this document. A server that advertises this extension SHOULD use i;unicode-casemap as the default comparator. (Note that i;unicode-casemap is the default comparator for I18NLEVEL=1, but not necessarily the default for I18NLEVEL=2.) The selection of the default comparator MAY be adjustable by the server administrator, and MAY be sensitive to the current user. Once the IMAP connection enters authenticated state, the default comparator MUST remain static for the remainder of that connection. Note that since SEARCH uses the substring operation, IMAP servers can only implement collations that offer the substring operation (see [RFC4790], Section 4.2.2). Since SORT uses the ordering operation (which in turn uses the equality operation), IMAP servers that advertise the SORT extension can only implement collations that offer all three operations (see [RFC4790], Sections 4.2.2-4.2.4). Newman, et al. Standards Track [Page 10] RFC 5255 IMAP Internationalization June 2008 If the active collation does not provide the operations needed by an IMAP command, the server MUST respond with a tagged BAD. 4.5. Compatibility Notes Several server implementations deployed prior to the publication of this specification comply with I18NLEVEL=1 (see Section 4.3), but do not advertise that. Other legacy servers use the i;ascii-casemap comparator (see [RFC4790]). There is no good way for a client to know which comparator a legacy server uses. If the client has to assume the worst, it may end up doing expensive local operations to obtain i;unicode-casemap comparisons even though the server implements it. Legacy server implementations which comply with I18NLEVEL=1 should be updated to advertise I18NLEVEL=1. All server implementations should eventually be updated to comply with the I18NLEVEL=2 extension. 4.6. Comparators and Character Encodings RFC 3501, Section 6.4.4, says: In all search keys that use strings, a message matches the key if the string is a substring of the field. The matching is case-insensitive. When performing the SEARCH operation, the active comparator is applied instead of the case-insensitive matching specified above. An IMAP server which performs collation operations (e.g., as part of commands such as SEARCH, SORT, and THREAD) does so according to the following procedure: (a) MIME encoding (for example, see [RFC2047] for headers and [RFC2045] for body parts) MUST be removed in the texts being collated. If MIME encoding removal fails for a message (e.g., a body part of the message has an unsupported Content-Transfer-Encoding, uses characters not allowed by the Content-Transfer-Encoding, etc.), the collation of this message is undefined by this specification, and is handled in an implementation-dependent manner. (b) The decoded text from (a) MUST be converted to the charset expected by the active comparator. Newman, et al. Standards Track [Page 11] RFC 5255 IMAP Internationalization June 2008 (c) For the substring operation: If step (b) failed (e.g., the text is in an unknown charset, contains a sequence that is not valid according in that charset, etc.), the original decoded text from (a) (i.e., before the charset conversion attempt) is collated using the i;octet comparator (see [RFC4790]). If step (b) was successful, the converted text from (b) is collated according to the active comparator. For the ordering operation: All strings that were successfully converted by step (b) are separated from all strings that failed step (b). Strings in each group are collated independently. All strings successfully converted by step (b) are then validated by the active comparator. Strings that pass validation are collated using the active comparator. All strings that either fail step (b) or fail the active collation's validity operation are collated (after applying step (a)) using the i;octet comparator (see [RFC4790]). The resulting sorted list is produced by appending all collated "failed" strings after all strings collated using the active comparator. Example: The following example demonstrates ordering of 4 different strings using the i;unicode-casemap [UCM] comparator. Strings are represented using hexadecimal notation used by ABNF [RFC5234]. (1) %xD0 %xC0 %xD0 %xBD %xD0 %xB4 %xD1 %x80 %xD0 %xB5 %xD0 %xB9 (labeled with charset=UTF-8) (2) %xD1 %x81 %xD0 %x95 %xD0 %xA0 %xD0 %x93 %xD0 %x95 %xD0 %x99 (labeled with charset=UTF-8) (3) %xD0 %x92 %xD0 %xB0 %xD1 %x81 %xD0 %xB8 %xD0 %xBB %xD0 %xB8 %xFF %xB9 (labeled with charset=UTF-8) (4) %xE1 %xCC %xC5 %xCB %xD3 %xC5 %xCA (labeled with charset=KOI8-R) Step (b) will convert string (4) to the following sequence of octets (in UTF-8): %xD0 %x90 %xD0 %xBB %xD0 %xB5 %xD0 %xBA %xD1 %x81 %xD0 %xB5 %xD0 %xB9 and will reject strings (1) and (3), as they contain octets not allowed in charset=UTF-8. Newman, et al. Standards Track [Page 12] RFC 5255 IMAP Internationalization June 2008 After that, using the i;unicode-casemap collation, string (4) will collate before string (2). Using the i;octet collation on the original strings, string (3) will collate before string (1). So the final ordering is as follows: (4) (2) (3) (1). If the substring operation (e.g., IMAP SEARCH) of the active comparator returns the "undefined" result (see Section 4.2.3 of [RFC4790]) for either the text specified in the SEARCH command or the message text, then the operation is repeated on the result of step (a) using the i;octet comparator. The ordering operation (e.g., IMAP SORT and THREAD) SHOULD collate the following together: strings encoded using unknown or invalid character encodings, strings in unrecognized charsets, and invalid input (as defined by the active collation). 4.7. COMPARATOR Command Arguments: Optional comparator order arguments. Response: A possible COMPARATOR response (see Section 4.8). Result: OK - Command completed NO - No matching comparator found BAD - Arguments invalid The COMPARATOR command is valid in authenticated and selected states. The COMPARATOR command is used to determine or change the active comparator. When issued with no arguments, it results in a COMPARATOR response indicating the currently active comparator. When issued with one or more comparator arguments, it changes the active comparator as directed. (If more than one installed comparator is matched by an argument, the first argument wins.) The COMPARATOR response lists all matching comparators if more than one matches the specified patterns. The argument "default" refers to the server's default comparator. Otherwise, each argument is a collation specification as defined in the Internet Application Protocol Comparator Registry [RFC4790]. < The client requests activating a Czech comparator if possible, or else a generic international comparator which it considers suitable for Czech. The server picks the first supported comparator. > Newman, et al. Standards Track [Page 13] RFC 5255 IMAP Internationalization June 2008 C: A001 COMPARATOR "cz;*" i;basic S: * COMPARATOR i;basic S: A001 OK Will use i;basic for collation 4.8. COMPARATOR Response Contents: The active comparator. An optional list of available matching comparators The COMPARATOR response occurs as a result of a COMPARATOR command. The first argument in the comparator response is the name of the active comparator. The second argument is a list of comparators which matched any of the arguments to the COMPARATOR command and is present only if more than one match is found. 4.9. BADCOMPARATOR Response Code This response code SHOULD be returned as a result of server failing an IMAP command (returning NO), when the server knows that none of the specified comparators match the requested comparator(s). 4.10. Formal Syntax The following syntax specification inherits ABNF [RFC5234] rules from IMAP4rev1 [RFC3501] and the Internet Application Protocol Comparator Registry [RFC4790]. command-auth =/ comparator-cmd resp-text-code =/ "BADCOMPARATOR" comparator-cmd = "COMPARATOR" *(SP comp-order-quoted) response-payload =/ comparator-data comparator-data = "COMPARATOR" SP comp-sel-quoted [SP "(" comp-id-quoted *(SP comp-id-quoted) ")"] comp-id-quoted = astring ; Once any literal wrapper or quoting is removed, this ; follows the collation-id rule from [RFC4790] comp-order-quoted = astring ; Once any literal wrapper or quoting is removed, this ; follows the collation-order rule from [RFC4790] Newman, et al. Standards Track [Page 14] RFC 5255 IMAP Internationalization June 2008 comp-sel-quoted = astring ; Once any literal wrapper or quoting is removed, this ; follows the collation-selected rule from [RFC4790] 5. Other IMAP Internationalization Issues The following sections provide an overview of various other IMAP internationalization issues. These issues are not resolved by this specification, but could be resolved by other standards work, such as that being done by the EAI working group (see [IMAP-EAI]). 5.1. Unicode Userids and Passwords IMAP4rev1 currently restricts the userid and password fields of the LOGIN command to US-ASCII. The "userid" and "password" fields of the IMAP LOGIN command are restricted to US-ASCII only until a future standards track RFC states otherwise. Servers are encouraged to validate both fields to make sure they conform to the formal syntax of UTF-8 and to reject the LOGIN command if that syntax is violated. Servers MAY reject the LOGIN command if either the "userid" or "password" field contains an octet with the highest bit set. When AUTHENTICATE is used, some servers may support userids and passwords in Unicode [RFC3490] since SASL (see [RFC4422]) allows that. However, such userids cannot be used as part of email addresses. 5.2. UTF-8 Mailbox Names The modified UTF-7 mailbox naming convention described in Section 5.1.3 of RFC 3501 is best viewed as an transition from the status quo in 1996 when modified UTF-7 was first specified. At that time, there was widespread unofficial use of local character sets such as ISO- 8859-1 and Shift-JIS for non-ASCII mailbox names, with resultant non-interoperability. The requirements in Section 5.1 of RFC 3501 are very important if we're ever going to be able to deploy UTF-8 mailbox names. Servers are encouraged to enforce them. 5.3. UTF-8 Domains, Addresses, and Mail Headers There is now an IETF standard for "Internationalizing Domain Names in Applications (IDNA)" [RFC3490]. While IMAP clients are free to support this standard, an argument can be made that it would be helpful to simple clients if the IMAP server could perform this conversion (the same argument would apply to MIME header encoding Newman, et al. Standards Track [Page 15] RFC 5255 IMAP Internationalization June 2008 [RFC2047]). However, it would be unwise to move forward with such work until the work in progress to define the format of international email addresses is complete. 6. IANA Considerations IANA added LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 to the IMAP4 Capabilities Registry. 7. Security Considerations The LANGUAGE extension makes a new command available in "Not Authenticated" state in IMAP. Some IMAP implementations run with root privilege when the server is in "Not Authenticated" state and do not revoke that privilege until after authentication is complete. Such implementations are particularly vulnerable to buffer overflow security errors at this stage and need to implement parsing of this command with extra care. A LANGUAGE command issued prior to activation of a security layer is subject to an active attack that suppresses or modifies the negotiation, and thus makes STARTTLS or authentication error messages more difficult to interpret. This is not a new attack as the error messages themselves are subject to active attack. Clients MUST re- issue the LANGUAGE command once a security layer is active, in order to prevent this attack from impacting subsequent protocol operations. LANGUAGE, I18NLEVEL=1, and I18NLEVEL=2 extensions use the UTF-8 charset; thus, the security considerations for UTF-8 [RFC3629] are relevant. However, neither uses UTF-8 for identifiers, so the most serious concerns do not apply. 8. Acknowledgements The LANGUAGE extension is based on a previous document by Mike Gahrns, a substantial portion of the text in that section was written by him. Many people have participated in discussions about an IMAP Language extension in the various fora of the IETF and Internet working groups, so any list of contributors is bound to be incomplete. However, the authors would like to thank Andrew McCown for early work on the original proposal, John Myers for suggestions regarding the namespace issue, along with Jutta Degener, Mark Crispin, Mark Pustilnik, Larry Osterman, Cyrus Daboo, Martin Duerst, Timo Sirainen, Ben Campbell, and Magnus Nystrom for their many suggestions that have been incorporated into this document. Initial discussion of the I18NLEVEL=2 extension involved input from Mark Crispin and other participants of the IMAP Extensions WG. Newman, et al. Standards Track [Page 16] RFC 5255 IMAP Internationalization June 2008 9. Relevant Sources of Documents for Internationalized IMAP Implementations This is a non-normative list of sources to consider when implementing i18n-aware IMAP software. o The LANGUAGE and I18NLEVEL=2 extensions to IMAP (this specification). o The 8-bit rules for mailbox naming in Section 5.1 of RFC 3501. o The Mailbox International Naming Convention in Section 5.1.3 of RFC 3501. o MIME [RFC2045] for message bodies. o MIME header encoding [RFC2047] for message headers. o The IETF EAI working group. o MIME Parameter Value and Encoded Word Extensions [RFC2231] for filenames. Quality IMAP server implementations will automatically combine multipart parameters when generating the BODYSTRUCTURE. There is also some deployed non-standard use of MIME header encoding inside double quotes for filenames. o IDNA [RFC3490] and punycode [RFC3492] for domain names (currently only relevant to IMAP clients). o The UTF-8 charset [RFC3629]. o The IETF policy on Character Sets and Languages [RFC2277]. 10. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and Languages", BCP 18, RFC 2277, January 1998. [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, May 1998. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. Newman, et al. Standards Track [Page 17] RFC 5255 IMAP Internationalization June 2008 [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC4422] Melnikov, A., Ed., and K. Zeilenga, Ed., "Simple Authentication and Security Layer (SASL)", RFC 4422, June 2006. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC4646] Phillips, A. and M. Davis, "Tags for Identifying Languages", BCP 47, RFC 4646, September 2006. [RFC4647] Phillips, A. and M. Davis, "Matching of Language Tags", BCP 47, RFC 4647, September 2006. [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet Application Protocol Collation Registry", RFC 4790, March 2007. [SORT] Crispin, M. and K. Murchison, "Internet Message Access Protocol - SORT and THREAD Extensions", RFC 5256, June 2008. [UCM] Crispin, M., "i;unicode-casemap - Simple Unicode Collation Algorithm", RFC 5051, October 2007. [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. 11. Informative References [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded Word Extensions: Character Sets, Languages, and Continuations", RFC 2231, November 1997. [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, "Internationalizing Domain Names in Applications (IDNA)", RFC 3490, March 2003. Newman, et al. Standards Track [Page 18] RFC 5255 IMAP Internationalization June 2008 [RFC3492] Costello, A., "Punycode: A Bootstring encoding of Unicode for Internationalized Domain Names in Applications (IDNA)", RFC 3492, March 2003. [METADATA] Daboo, C., "IMAP METADATA Extension", Work in Progress, April 2008. [IMAP-EAI] Resnick, P., and C. Newman, "IMAP Support for UTF-8", Work in Progress, November 2007. Authors' Addresses Chris Newman Sun Microsystems 3401 Centrelake Dr., Suite 410 Ontario, CA 91761 US EMail: chris.newman@sun.com Arnt Gulbrandsen Oryx Mail Systems GmbH Schweppermannstr. 8 D-81671 Muenchen Germany EMail: arnt@oryx.com Fax: +49 89 4502 9758 Alexey Melnikov Isode Limited 5 Castle Business Village, 36 Station Road, Hampton, Middlesex, TW12 2BX, UK EMail: Alexey.Melnikov@isode.com Newman, et al. Standards Track [Page 19] RFC 5255 IMAP Internationalization June 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Newman, et al. Standards Track [Page 20] ================================================ FILE: docs/rfcs/rfc5257.IMAP_ANNOTATE_extension.txt ================================================ Network Working Group C. Daboo Request for Comments: 5257 Apple Inc. Category: Experimental R. Gellens QUALCOMM Incorporated June 2008 Internet Message Access Protocol - ANNOTATE Extension Status of This Memo This memo defines an Experimental Protocol for the Internet community. It does not specify an Internet standard of any kind. Discussion and suggestions for improvement are requested. Distribution of this memo is unlimited. Abstract The ANNOTATE extension to the Internet Message Access Protocol permits clients and servers to maintain "meta data" for messages, or individual message parts, stored in a mailbox on the server. For example, this can be used to attach comments and other useful information to a message. It is also possible to attach annotations to specific parts of a message, so that, for example, they could be marked as seen, or important, or a comment added. Note that this document was the product of a WG that had good consensus on how to approach the problem. Nevertheless, the WG felt it did not have enough information on implementation and deployment hurdles to meet all of the requirements of a Proposed Standard. The IETF solicits implementations and implementation reports in order to make further progress. Implementers should be aware that this specification may change in an incompatible manner when going to Proposed Standard status. However, any incompatible changes will result in a new capability name being used to prevent problems with any deployments of the experimental extension. Daboo & Gellens Experimental [Page 1] RFC 5257 IMAP ANNOTATE Extension June 2008 Table of Contents 1. Introduction and Overview .......................................3 2. Conventions Used in This Document ...............................4 3. Data Model ......................................................4 3.1. Overview ...................................................4 3.2. Namespace of Entries and Attributes ........................4 3.2.1. Entry Names .........................................5 3.2.2. Attribute Names .....................................7 3.3. Private Versus Shared ......................................7 3.4. Access Control .............................................8 3.5. Access to Standard IMAP Flags and Keywords ................11 4. IMAP Protocol Changes ..........................................11 4.1. General Considerations ....................................11 4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands .......12 4.3. ANNOTATION Message Data Item in FETCH Command .............12 4.4. ANNOTATION Message Data Item in FETCH Response ............14 4.5. ANNOTATION Message Data Item in STORE .....................16 4.6. ANNOTATION Interaction with COPY ..........................18 4.7. ANNOTATION Message Data Item in APPEND ....................18 4.8. ANNOTATION Criterion in SEARCH ............................19 4.9. ANNOTATION Key in SORT ....................................20 4.10. New ACL Rights ...........................................21 5. Formal Syntax ..................................................21 6. IANA Considerations ............................................23 6.1. Entry and Attribute Registration Template .................23 6.2. Entry Registrations .......................................24 6.2.1. /comment ...........................................24 6.2.2. /flags .............................................24 6.2.3. /altsubject ........................................25 6.2.4. //comment ............................25 6.2.5. //flags/seen .........................26 6.2.6. //flags/answered .....................26 6.2.7. //flags/flagged ......................27 6.2.8. //flags/forwarded ....................27 6.3. Attribute Registrations ...................................28 6.3.1. value ..............................................28 6.3.2. size ...............................................28 6.4. Capability Registration ...................................28 7. Internationalization Considerations ............................29 8. Security Considerations ........................................29 9. References .....................................................29 9.1. Normative References ......................................29 9.2. Informative References ....................................30 10. Acknowledgments ...............................................30 Daboo & Gellens Experimental [Page 2] RFC 5257 IMAP ANNOTATE Extension June 2008 1. Introduction and Overview The ANNOTATE extension is present in any IMAP [RFC3501] implementation that returns "ANNOTATE-EXPERIMENT-1" as one of the supported capabilities in the CAPABILITY response. This extension makes the following changes to the IMAP protocol: a. adds a new ANNOTATION message data item for use in FETCH. b. adds a new ANNOTATION message data item for use in STORE. c. adds a new ANNOTATION search criterion for use in SEARCH. d. adds a new ANNOTATION sort key for use in the SORT extension. e. adds a new ANNOTATION data item for use in APPEND. f. adds a new requirement on the COPY command. g. adds a new ANNOTATE parameter for use with the SELECT/EXAMINE commands. h. adds two new response codes to indicate store failures of annotations. i. adds a new untagged response code for the SELECT or EXAMINE commands to indicate the maximum sized annotation that can be stored. j. adds a new Access Control List (ACL) "bit" for use with the ACL extensions [RFC2086] and [RFC4314]. The data model used for the storage of annotations is based on the Application Configuration Access Protocol [RFC2244]. Note that there is no inheritance in annotations. If a server supports annotations, then it MUST store all annotation data permanently, i.e., there is no concept of "session only" annotations that would correspond to the behavior of "session" flags as defined in the IMAP base specification. In order to provide optimum support for a disconnected client (one that needs to synchronize annotations for use when offline), servers SHOULD also support the Conditional STORE [RFC4551] extension. The rest of this document describes the data model and protocol changes more rigorously. Daboo & Gellens Experimental [Page 3] RFC 5257 IMAP ANNOTATE Extension June 2008 2. Conventions Used in This Document The examples in this document use "C:" and "S:" to indicate lines sent by the client and server, respectively. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. 3. Data Model 3.1. Overview The data model for annotations in ANNOTATE uses a uniquely named entry that contains a set of standard attributes. Thus, a single coherent unit of "meta data" for a message is stored as a single entry, made up of several attributes. For example, a comment annotation added to a message has an entry name of "/comment". This entry is composed of several attributes such as "value", "size", etc., that contain the properties and data of the entry. The protocol changes to IMAP, described below, allow a client to access or change the values of any attribute in any entry in a message annotation, assuming it has sufficient access rights to do so (see Section 3.4 for specifics). 3.2. Namespace of Entries and Attributes A message may contain zero or more annotations, each of which is a single uniquely named entry. Each entry has a hierarchical name, with each component of the name separated by a slash ("/"). An entry name MUST NOT contain two consecutive "/" characters and MUST NOT end with a "/" character. Each entry is made up of a set of attributes. Each attribute has a hierarchical name, with each component of the name separated by a period ("."). An attribute name MUST NOT contain two consecutive "." characters and MUST NOT end with a "." character. The value of an attribute is "NIL" (has no value), or is a string of zero or more octets. Entry and attribute names MUST NOT contain asterisk ("*") or percent ("%") characters, and MUST NOT contain non-ASCII characters or the NULL octet. Invalid entry or attribute names result in a BAD response in any IMAP commands where they are used. Daboo & Gellens Experimental [Page 4] RFC 5257 IMAP ANNOTATE Extension June 2008 Attribute names MUST NOT contain any hierarchical components with the names "priv" or "shared", as those have special meaning (see Section 3.3). Entry and attribute names are case-sensitive. Use of control or punctuation characters in entry and attribute names is strongly discouraged. This specification defines an initial set of entry and attribute names available for use in message annotations. In addition, an extension mechanism is described to allow additional names to be added as needed. 3.2.1. Entry Names Entry names MUST be specified in a standards track or IESG approved experimental RFC, or fall under the vendor namespace. See Section 6.1 for the registration template. / Defines the top-level of entries associated with an entire message. This entry itself does not contain any attributes. All entries that start with a numeric character ("0" - "9") refer to an annotation on a specific body part. All other entries are for annotations on the entire message. /comment Defines a comment or note associated with an entire message. /flags This entry hierarchy is reserved for future use. /altsubject Contains text supplied by the message recipient to be used by the client, instead of the original message Subject. /vendor/ Defines the top-level of entries associated with an entire message as created by a particular product of some vendor. These sub- entries can be used by vendors to provide client-specific annotations. The vendor-token MUST be registered with IANA, using the [RFC2244] vendor subtree registry. / Defines the top-level of entries associated with a specific body part of a message. This entry itself does not contain any attributes. The section-part is a numeric part specifier. Its Daboo & Gellens Experimental [Page 5] RFC 5257 IMAP ANNOTATE Extension June 2008 syntax is the same as the section-part ABNF element defined in [RFC3501]. The server MUST return a BAD response if the client uses an incorrect part specifier (either incorrect syntax or a specifier referring to a non-existent part). The server MUST return a BAD response if the client uses an empty part specifier (which is used in IMAP to represent the entire message). //comment Defines a comment or note associated with a specific body part of a message. //flags Defines the top-level of entries associated with the flag state for a specific body part of a message. All sub-entries are maintained entirely by the client. There is no implicit change to any flag by the server. //flags/seen This is similar to the IMAP \Seen flag, except it applies to only the body part referenced by the entry. //flags/answered This is similar to the IMAP \Answered flag, except it applies to only the body part referenced by the entry. //flags/flagged This is similar to the IMAP \Flagged flag, except it applies to only the body part referenced by the entry. //flags/forwarded This is similar to the IMAP $Forwarded keyword, except it applies to only the body part referenced by the entry. Defines flags for a specific body part of a message. The "value" attribute of each of the entries described above must be either "1", "0", or "NIL". "1" corresponds to the flag being set. //vendor/ Defines the top-level of entries associated with a specific body part of a message as created by a particular product of some vendor. This entry can be used by vendors to provide client specific annotations. The vendor-token MUST be registered with IANA. Daboo & Gellens Experimental [Page 6] RFC 5257 IMAP ANNOTATE Extension June 2008 3.2.2. Attribute Names Attribute names MUST be specified in a standards track or IESG approved experimental RFC. See Section 6.1 for the registration template. All attribute names implicitly have a ".priv" and a ".shared" suffix that maps to private and shared versions of the entry. Searching or fetching without using either suffix will include both. The client MUST specify either a ".priv" or ".shared" suffix when storing an annotation or sorting on annotations. value A string or binary data representing the value of the annotation. To delete an annotation, the client can store "NIL" into the value. If the client requests the value attribute for a non- existent entry, then the server MUST return "NIL" for the value. The content represented by the string is determined by the content-type used to register the entry (see Section 6.1 for entry registration templates). Where applicable, the registered content-type MUST include a charset parameter. Text values SHOULD use the utf-8 [RFC3629] character set. Note that binary data (data which may contain the NULL octet) is allowed (e.g., for storing images), and this extension uses the "literal8" syntax element [RFC4466] to allow such data to be written to or read from the server. size The size of the value, in octets. Set automatically by the server, read-only to clients. If the client requests the size attribute for a non-existent entry, then the server MUST return "0" (zero) for the size. 3.3. Private Versus Shared Some IMAP mailboxes are private, accessible only to the owning user. Other mailboxes are not, either because the owner has set an ACL [RFC4314] that permits access by other users, or because it is a shared mailbox. This raises the issue of shared versus private annotations. If all annotations are private, it is then impossible to have annotations in a shared or otherwise non-private mailbox be visible to other users. This eliminates what could be a useful aspect of annotations in a shared environment. An example of such use is a shared IMAP folder containing bug reports. Engineers may want to use Daboo & Gellens Experimental [Page 7] RFC 5257 IMAP ANNOTATE Extension June 2008 annotations to add information to existing messages, indicate assignments, status, etc. This use requires shared annotations. If all annotations are shared, it is impossible to use annotations for private notes on messages in shared mailboxes. Also, modifying an ACL to permit access to a mailbox by other users may unintentionally expose private information. There are also situations in which both shared and private annotations are useful. For example, an administrator may want to set shared annotations on messages in a shared folder, which individual users may wish to supplement with additional notes. If shared and private annotations are to coexist, we need a clear way to differentiate them. Also, it should be as easy as possible for a client to access both and not overlook either. There is also a danger in allowing a client to store an annotation without knowing if it is shared or private. This document proposes two standard suffixes for all attributes: ".shared" and ".priv". A SEARCH or FETCH command that specifies neither, uses both. STORE, APPEND, and SORT commands MUST explicitly use ".priv" or ".shared" suffixes. If the ANNOTATE extension is present, support for shared annotations in servers is REQUIRED, while support for private annotations in servers is OPTIONAL. This recognizes the fact that support for private annotations may introduce a significant increase in complexity to a server in terms of tracking ownership of the annotations, how quota is determined for users based on their own annotations, etc. Clients that support the ANNOTATE extension MUST handle both shared and private annotations. 3.4. Access Control A user needs to have appropriate rights in order to read or write ".priv" or ".shared" annotation values. How those rights are calculated depends on whether or not the ACL [RFC2086] extension or its update [RFC4314] is present. If a client attempts to store or fetch an annotation to which they do not have the appropriate rights, the server MUST respond with a NO response. When the ACL extension is not present, access to annotation values is governed by the nature of the selected state, in particular whether the mailbox was SELECTED or EXAMINED in READ-ONLY or READ-WRITE mode. Daboo & Gellens Experimental [Page 8] RFC 5257 IMAP ANNOTATE Extension June 2008 When the ACL extension is present, the server MUST recognize the new ACL "n" right, in addition to the ones defined by the ACL extension itself. For ".priv" annotation values, the "r" right controls both read and write access. When it is on, access to ".priv" annotations is allowed; when it is off, access to ".priv" annotations is disallowed. For ".shared" annotation values, the "r" right controls read access. When it is on, ".shared" annotations can be read; when it is off, ".shared" annotations cannot be read. For ".shared" annotation values, the "n" right controls write access. When it is on, ".shared" annotations can be changed or created through either a STORE or APPEND command; when it is off, ".shared" annotations cannot be changed or created. The "n" right constitutes a "shared flag right" as defined in Section 6.2 of [RFC4314]. Daboo & Gellens Experimental [Page 9] RFC 5257 IMAP ANNOTATE Extension June 2008 A summary of all the access control restrictions is tabulated below +---------------+---------------+-----------------------------------+ | Server Type | Action on | Type of mailbox | | | annotation | | +===============+===============+===================================+ | | | | | | read .priv | Any mailbox that can be SELECTED | | | values | or EXAMINED. | | | | | | +---------------+-----------------------------------+ | | | | | | write .priv | Any SELECTED [READ-WRITE] mailbox.| | | values | SELECTED [READ-ONLY] mailboxes MAY| | Server | | also permit writes. | | without | | | | ACL Extension +---------------+-----------------------------------+ | | | | | | read .shared | Any mailbox that can be SELECTED | | | values | or EXAMINED. | | | | | | +---------------+-----------------------------------+ | | | | | | write .shared | Any mailbox that can be SELECTED | | | values | or EXAMINED and is [READ-WRITE]. | | | | | +---------------+---------------+-----------------------------------+ | | | | | | read .priv | Any mailbox with the "r" | | | values | ACL right. | | | | | | +---------------+-----------------------------------+ | | | | | | write .priv | Any mailbox with the "r" | | Server | values | ACL right. | | with | | | | ACL Extension +---------------+-----------------------------------+ | | | | | | read .shared | Any mailbox with the "r" | | | values | ACL right. | | | | | | +---------------+-----------------------------------+ | | | | | | write .shared | Any mailbox with the "n" | | | values | ACL right. | | | | | +---------------+---------------+-----------------------------------+ Daboo & Gellens Experimental [Page 10] RFC 5257 IMAP ANNOTATE Extension June 2008 3.5. Access to Standard IMAP Flags and Keywords Due to the ambiguity of how private and shared values would map to the base IMAP flag and keyword values, the ANNOTATE extension does not expose IMAP flags or keywords as entries. However, the /flags annotation entry is reserved for future use and MUST NOT be used by clients or servers supporting this extension. Clients that need to implement shared and private "flags" can create their own annotation entries for those, completely bypassing the base IMAP flag/keyword behavior. 4. IMAP Protocol Changes 4.1. General Considerations Servers may be able to offer only a limited level of support for annotations in mailboxes, and it is useful for clients to be able to know what level of support is available. Servers MUST return an ANNOTATIONS response code during the SELECT or EXAMINE command for a mailbox to indicate the level of support. Possible data items used with the ANNOTATIONS response code are: "NONE" - this indicates that the mailbox being selected does not support annotations at all. Clients MUST NOT attempt to use annotation extensions in commands for this mailbox. "READ-ONLY" - this indicates that the annotations supported by the mailbox cannot be changed by the client. Clients MUST NOT attempt to store annotations on any messages in a mailbox with this response code. "NOPRIVATE" - this indicates that the server does not support private annotations on the mailbox. Only shared annotations are supported. Clients SHOULD only attempt to read or store annotations attributes with the ".shared" suffix. If a client uses an attribute with the ".priv" suffix in a FETCH command, then servers should return the attribute value in the FETCH response as "NIL". If a client uses an attribute with the ".priv" suffix in a STORE command (or an APPEND command targeted at the mailbox), then the server MUST return a NO response. numeric values - if servers support writable annotations, then the server MUST indicate the maximum size in octets for an annotation value by providing the maximum size value in the response code. Clients MUST NOT store annotation values of a size greater than the amount indicated by the server. Servers MUST accept a minimum Daboo & Gellens Experimental [Page 11] RFC 5257 IMAP ANNOTATE Extension June 2008 annotation data size of at least 1024 octets if annotations can be written. In addition, the server MAY limit the total number of annotations for a single message. However, the server MUST provide a minimum annotation count per message of at least 10. 4.2. ANNOTATE Parameter with the SELECT/EXAMINE Commands The ANNOTATE extension defines a single optional SELECT parameter [RFC4466] "ANNOTATE", which is used to turn on unsolicited responses for annotations as described in Section 4.4. This optional parameter results in a per-mailbox state change, i.e., it must be used in each SELECT/EXAMINE command in order to be effective, irrespective of whether it was used in a previous SELECT/EXAMINE during the same session. Example: C: a SELECT INBOX (ANNOTATE) S: * FLAGS (\Answered \Flagged \Draft \Deleted \Seen) S: * OK [PERMANENTFLAGS (\Answered \Flagged \Draft \Deleted \Seen \*)] S: * 10268 EXISTS S: * 1 RECENT S: * OK [UNSEEN 10268] S: * OK [UIDVALIDITY 890061587] S: * OK [UIDNEXT 34643] S: * OK [ANNOTATIONS 20480 NOPRIVATE] S: a OK [READ-WRITE] Completed In the above example, a SELECT command with the ANNOTATE parameter is issued. The response from the server includes the required ANNOTATIONS response that indicates that the server supports annotations up to a maximum size of 20480 octets, and does not support private annotations (only shared). 4.3. ANNOTATION Message Data Item in FETCH Command This extension adds an ANNOTATION message data item to the FETCH command. This allows clients to retrieve annotations for a range of messages in the currently selected mailbox. ANNOTATION The ANNOTATION message data item, when used by the client in the FETCH command, takes an entry specifier and an attribute specifier. Daboo & Gellens Experimental [Page 12] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a FETCH 1 (ANNOTATION (/comment value)) S: * 1 FETCH (ANNOTATION (/comment (value.priv "My comment" value.shared "Group note"))) S: a OK Fetch complete In the above example, the content of the "value" attribute for the "/comment" entry is requested by the client and returned by the server. Since neither ".shared" nor ".priv" was specified, both are returned. "*" and "%" wild card characters can be used in entry specifiers to match one or more characters at that position, with the exception that "%" does not match the "/" hierarchy delimiter. Thus, an entry specifier of "/%" matches entries such as "/comment" and "/altsubject", but not "/1/comment". Example: C: a UID FETCH 1123 (UID ANNOTATION (/* (value.priv size.priv))) S: * 12 FETCH (UID 1123 ANNOTATION (/comment (value.priv "My comment" size.priv "10") /altsubject (value.priv "Rhinoceroses!" size.priv "13") /vendor/foobar/label.priv (value.priv "label43" size.priv "7") /vendor/foobar/personality (value.priv "Tallulah Bankhead" size.priv "17"))) S: a OK Fetch complete In the above example, the contents of the private "value" and "size" attributes for any entries in the "/" hierarchy are requested by the client and returned by the server. Daboo & Gellens Experimental [Page 13] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a FETCH 1 (ANNOTATION (/% value.shared)) S: * 1 FETCH (ANNOTATION (/comment (value.shared "Patch Mangler") /altsubject (value.shared "Patches? We don't need no steenkin patches!"))) S: a OK Fetch complete In the above example, the contents of the shared "value" attributes for entries at the top level only of the "/" hierarchy are requested by the client and returned by the server. Entry and attribute specifiers can be lists of atomic specifiers, so that multiple items of each type may be returned in a single FETCH command. Example: C: a FETCH 1 (ANNOTATION ((/comment /altsubject) value.priv)) S: * 1 FETCH (ANNOTATION (/comment (value.priv "What a chowder-head") /altsubject (value.priv "How to crush beer cans"))) S: a OK Fetch complete In the above example, the contents of the private "value" attributes for the two entries "/comment" and "/altsubject" are requested by the client and returned by the server. 4.4. ANNOTATION Message Data Item in FETCH Response The ANNOTATION message data item in the FETCH response displays information about annotations in a message. ANNOTATION parenthesized list The response consists of a list of entries, each of which have a list of attribute-value pairs. Daboo & Gellens Experimental [Page 14] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a FETCH 1 (ANNOTATION (/comment value)) S: * 1 FETCH (ANNOTATION (/comment (value.priv "My comment" value.shared NIL))) S: a OK Fetch complete In the above example, a single entry with a single attribute-value pair is returned by the server. Since the client did not specify a ".shared" or ".priv" suffix, both are returned. Only the private attribute has a value (the shared value is "NIL"). Example: C: a FETCH 1 (ANNOTATION ((/comment /altsubject) value)) S: * 1 FETCH (ANNOTATION (/comment (value.priv "My comment" value.shared NIL) /altsubject (value.priv "My subject" value.shared NIL))) S: a OK Fetch complete In the above example, two entries, each with a single attribute- value pair, are returned by the server. Since the client did not specify a ".shared" or ".priv" suffix, both are returned. Only the private attributes have values; the shared attributes are "NIL". Example: C: a FETCH 1 (ANNOTATION (/comment (value size))) S: * 1 FETCH (ANNOTATION (/comment (value.priv "My comment" value.shared NIL size.priv "10" size.shared "0"))) S: a OK Fetch complete In the above example, a single entry with two attribute-value pairs is returned by the server. Since the client did not specify a ".shared" or ".priv" suffix, both are returned. Only the private attributes have values; the shared attributes are "NIL". Daboo & Gellens Experimental [Page 15] RFC 5257 IMAP ANNOTATE Extension June 2008 Servers SHOULD send ANNOTATION message data items in unsolicited FETCH responses if an annotation entry is changed by a third-party, and the ANNOTATE select parameter was used. This allows servers to keep clients updated with changes to annotations by other clients. Unsolicited ANNOTATION responses MUST NOT include ANNOTATION data values -- only the entry name of the ANNOTATION that has changed. This restriction avoids sending ANNOTATION data values (which may be large) to a client unless the client explicitly asks for the value. Example: C: a STORE 1 +FLAGS (\Seen) S: * 1 FETCH (FLAGS (\Seen)) ANNOTATION (/comment)) S: a OK STORE complete In the above example, an unsolicited ANNOTATION response is returned during a STORE command. The unsolicited response contains only the entry name of the annotation that changed, and not its value. 4.5. ANNOTATION Message Data Item in STORE ANNOTATION Sets the specified list of entries by adding or replacing the specified attributes with the values provided. Clients can use "NIL" for values of attributes it wants to remove from entries. The ANNOTATION message data item used with the STORE command has an implicit ".SILENT" behavior. This means the server does not generate an untagged FETCH in response to the STORE command and assumes that the client updates its own cache if the command succeeds. Though note, that if the Conditional STORE extension [RFC4551] is present, then an untagged FETCH response with a MODSEQ data item will be returned by the server as required by [RFC4551]. If the server is unable to store an annotation because the size of its value is too large, the server MUST return a tagged NO response with a "[ANNOTATE TOOBIG]" response code. If the server is unable to store a new annotation because the maximum number of allowed annotations has already been reached, the server MUST return a tagged NO response with a "[ANNOTATE TOOMANY]" response code. Daboo & Gellens Experimental [Page 16] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a STORE 1 ANNOTATION (/comment (value.priv "My new comment")) S: a OK Store complete In the above example, the entry "/comment" is created (if not already present). Its private attribute "value" is created if not already present, or replaced if it exists. "value.priv" is set to "My new comment". Example: C: a STORE 1 ANNOTATION (/comment (value.shared NIL)) S: a OK Store complete In the above example, the shared "value" attribute of the entry "/comment" is removed by storing "NIL" into the attribute. Multiple entries can be set in a single STORE command by listing entry-attribute-value pairs in the list. Example: C: a STORE 1 ANNOTATION (/comment (value.priv "Get tix Tuesday") /altsubject (value.priv "Wots On")) S: a OK Store complete In the above example, the entries "/comment" and "/altsubject" are created (if not already present) and the private attribute "value" is created or replaced for each entry. Multiple attributes can be set in a single STORE command by listing multiple attribute-value pairs in the entry list. Daboo & Gellens Experimental [Page 17] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a STORE 1 ANNOTATION (/comment (value.priv "My new comment" value.shared "foo's bar")) S: a OK Store complete In the above example, the entry "/comment" is created (if not already present) and the private and shared "value" attributes are created if not already present, or replaced if they exist. 4.6. ANNOTATION Interaction with COPY The COPY command can be used to move messages from one mailbox to another on the same server. Servers that support the ANNOTATION extension MUST, for each message being copied, copy all ".priv" annotation data for the current user only, and all ".shared" annotation data along with the message to the new mailbox. The only exceptions to this are if the destination mailbox permissions are such that either the ".priv" or ".shared" annotations are not allowed, or if the destination mailbox is of a type that does not support annotations or does not support storing of annotations (a mailbox that returns a "NONE" or "READ-ONLY" response code in its ANNOTATIONS response), or if the destination mailbox cannot support the size of an annotation because it exceeds the ANNOTATIONS value. Servers MUST NOT copy ".priv" annotation data for users other than the current user. 4.7. ANNOTATION Message Data Item in APPEND ANNOTATION Sets the specified list of entries and attributes in the resulting message. The APPEND command can include annotations for the message being appended via the addition of a new append data item [RFC4466]. The new data item can also be used with the multi-append [RFC3502] extension that allows multiple messages to be appended via a single APPEND command. Daboo & Gellens Experimental [Page 18] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a APPEND drafts ANNOTATION (/comment (value.priv "Don't send until I say so")) {310} S: + Ready for literal data C: MIME-Version: 1.0 ... C: S: a OK APPEND completed In the above example, a comment with a private value is added to a new message appended to the mailbox. The ellipsis represents the bulk of the message. 4.8. ANNOTATION Criterion in SEARCH ANNOTATION The ANNOTATION criterion for the SEARCH command allows a client to search for a specified string in the value of an annotation entry of a message. Messages that have annotations with entries matching , attributes matching , and the specified string in their values are returned in the SEARCH results. The "*" character can be used in the entry name field to match any content in those items. The "%" character can be used in the entry name field to match a single level of hierarchy only. Only the "value", "value.priv", and "value.shared" attributes can be searched. Clients MUST NOT specify an attribute other than either "value", "value.priv", or "value.shared". Servers MUST return a BAD response if the client tries to search any other attribute. Example: C: a SEARCH ANNOTATION /comment value "IMAP4" S: * SEARCH 2 3 5 7 11 13 17 19 23 S: a OK Search complete In the above example, the message numbers of any messages containing the string "IMAP4" in the shared or private "value" attribute of the "/comment" entry are returned in the search results. Daboo & Gellens Experimental [Page 19] RFC 5257 IMAP ANNOTATE Extension June 2008 Example: C: a SEARCH ANNOTATION * value.priv "IMAP4" S: * SEARCH 1 2 3 5 8 13 21 34 S: a OK Search complete In the above example, the message numbers of any messages containing the string "IMAP4" in the private "value" attribute of any entry are returned in the search results. 4.9. ANNOTATION Key in SORT ANNOTATION The ANNOTATION criterion for the SORT command [RFC5256] instructs the server to return the sequence numbers or Unique Identifiers (UIDs) of messages in a mailbox, sorted using the values of the specified annotations. The ANNOTATION criterion is available if the server returns both ANNOTATE-EXPERIMENT-1 and SORT as supported capabilities in the CAPABILITY command response. Messages are sorted using the values of the attributes in the entries. Clients MUST provide either the ".priv" or ".shared" suffix to the attribute name to ensure that the server knows which specific value to sort on. Only the "value.priv" and "value.shared" attributes can be used for sorting. Clients MUST NOT specify an attribute other than either "value.priv" or "value.shared". Servers MUST return a BAD response if the client tries to sort on any other attribute. When either "value.priv" or "value.shared" is being sorted, the server MUST use the character set value specified in the SORT command to determine the appropriate sort order. Example: C: a SORT (ANNOTATION /altsubject value.shared) UTF-8 ALL S: * SORT 2 3 4 5 1 11 10 6 7 9 8 S: a OK Sort complete In the above example, the message numbers of all messages are returned, sorted according to the shared "value" attribute of the "/altsubject" entry. Daboo & Gellens Experimental [Page 20] RFC 5257 IMAP ANNOTATE Extension June 2008 Note that the ANNOTATION sort key must include a fully specified entry -- wild cards are not allowed. 4.10. New ACL Rights As discussed in Section 3.4, this extension adds a new "n" right to the list of rights provided by the ACL extensions [RFC2086] and [RFC4314]. 5. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234]. Non-terminals referenced but not defined below are as defined by [RFC3501] with the new definitions in [RFC4466] superseding those in [RFC3501]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. ann-size = "NONE" / (("READ-ONLY" / nz-number) [SP "NOPRIVATE"]) ; response codes indicating the level of ; support for annotations in a mailbox append-ext =/ att-annotate ; modifies [RFC3501] extension behaviour att-annotate = "ANNOTATION" SP "(" entry-att *(SP entry-att) ")" att-search = "value" / "value.priv" / "value.shared" ; the only attributes that can be searched att-sort = "value.priv" / "value.shared" ; the only attributes that can be sorted att-value = attrib SP value attrib = astring ; dot-separated attribute name ; MUST NOT contain "*" or "%" Daboo & Gellens Experimental [Page 21] RFC 5257 IMAP ANNOTATE Extension June 2008 attribs = attrib / "(" attrib *(SP attrib) ")" ; one or more attribute specifiers capability =/ "ANNOTATE-EXPERIMENT-1" ; defines the capability for this extension entries = entry-match / "(" entry-match *(SP entry-match) ")" entry = astring ; slash-separated path to entry ; MUST NOT contain "*" or "%" entry-att = entry SP "(" att-value *(SP att-value) ")" entry-match = list-mailbox ; slash-separated path to entry ; MAY contain "*" or "%" for use as wild cards fetch-att =/ "ANNOTATION" SP "(" entries SP attribs ")" ; modifies original IMAP fetch-att msg-att-dynamic =/ "ANNOTATION" SP ( "(" entry-att *(SP entry-att) ")" / "(" entry *(SP entry) ")" ) ; extends FETCH response with annotation data resp-text-code =/ "ANNOTATE" SP "TOOBIG" / "ANNOTATE" SP "TOOMANY" / "ANNOTATIONS" SP ann-size ; new response codes search-key =/ "ANNOTATION" SP entry-match SP att-search SP value ; modifies original IMAP search-key select-param =/ "ANNOTATE" ; defines the select parameter used with ; ANNOTATE extension sort-key =/ "ANNOTATION" SP entry SP att-sort ; modifies original sort-key store-att-flags =/ att-annotate ; modifies original IMAP STORE command value = nstring / literal8 Daboo & Gellens Experimental [Page 22] RFC 5257 IMAP ANNOTATE Extension June 2008 6. IANA Considerations Entry names MUST be specified in a standards track or IESG approved experimental RFC, or fall under the vendor namespace. Vendor names MUST be registered. Attribute names MUST be specified in a standards track or IESG approved experimental RFC. Each entry registration MUST include a content-type that is used to indicate the nature of the annotation value. Where applicable, a charset parameter MUST be included with the content-type. 6.1. Entry and Attribute Registration Template To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [] Entry [] Attribute Name: ______________________________ Description: _______________________ ____________________________________ ____________________________________ Content-Type:_______________________ Contact person: ____________________ email: ____________________ Daboo & Gellens Experimental [Page 23] RFC 5257 IMAP ANNOTATE Extension June 2008 6.2. Entry Registrations The following templates specify the IANA registrations of annotation entries specified in this document. 6.2.1. /comment To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: /comment Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name 6.2.2. /flags To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: /flags Description: Reserved entry hierarchy. Content-Type: - Contact person: Cyrus Daboo email: cyrus@daboo.name Daboo & Gellens Experimental [Page 24] RFC 5257 IMAP ANNOTATE Extension June 2008 6.2.3. /altsubject To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: /altsubject Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name 6.2.4. //comment To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: //comment Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name Daboo & Gellens Experimental [Page 25] RFC 5257 IMAP ANNOTATE Extension June 2008 6.2.5. //flags/seen To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: //flags/seen Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name 6.2.6. //flags/answered To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: //flags/answered Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name Daboo & Gellens Experimental [Page 26] RFC 5257 IMAP ANNOTATE Extension June 2008 6.2.7. //flags/flagged To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: //flags/flagged Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name 6.2.8. //flags/forwarded To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [X] Entry [] Attribute Name: //flags/forwarded Description: Defined in IMAP ANNOTATE extension document. Content-Type: text/plain; charset=utf-8 Contact person: Cyrus Daboo email: cyrus@daboo.name Daboo & Gellens Experimental [Page 27] RFC 5257 IMAP ANNOTATE Extension June 2008 6.3. Attribute Registrations The following templates specify the IANA registrations of annotation attributes specified in this document. 6.3.1. value To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [] Entry [X] Attribute Name: value Description: Defined in IMAP ANNOTATE extension document. Contact person: Cyrus Daboo email: cyrus@daboo.name 6.3.2. size To: iana@iana.org Subject: IMAP Annotate Registration Please register the following IMAP Annotate item: [] Entry [X] Attribute Name: size Description: Defined in IMAP ANNOTATE extension document. Contact person: Cyrus Daboo email: cyrus@daboo.name 6.4. Capability Registration This document registers "ANNOTATE-EXPERIMENT-1" as an IMAPEXT capability. Daboo & Gellens Experimental [Page 28] RFC 5257 IMAP ANNOTATE Extension June 2008 7. Internationalization Considerations Annotations may contain values that include text strings, and both searching and sorting are possible with annotations. Servers MUST follow standard IMAP text normalization, character set conversion, and collation rules when such operations are carried out, as would be done for other textual fields being searched or sorted on. 8. Security Considerations Annotations whose values are intended to remain private MUST be stored in ".priv" values instead of ".shared" values, which may be accessible to other users. Excluding the above issues, the ANNOTATE extension does not raise any security considerations that are not present in the base IMAP protocol; these issues are discussed in [RFC3501]. 9. References 9.1. Normative References [RFC2086] Myers, J., "IMAP4 ACL extension", RFC 2086, January 1997. [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC2244] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC3502] Crispin, M., "Internet Message Access Protocol (IMAP) - MULTIAPPEND Extension", RFC 3502, March 2003. [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. Daboo & Gellens Experimental [Page 29] RFC 5257 IMAP ANNOTATE Extension June 2008 [RFC5256] Crispin, M. and K. Murchison, "Internet Message Access Protocol - SORT and THREAD Extensions", RFC 5256, June 2008. 9.2. Informative References [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. 10. Acknowledgments Many thanks to Chris Newman for his detailed comments on the first draft of this document, and to the participants at the ACAP working dinner in Pittsburgh. The participants of the IMAPext working group made significant contributions to this work. Authors' Addresses Cyrus Daboo Apple Inc. 1 Infinite Loop Cupertino, CA 95014 USA EMail: cyrus@daboo.name URI: http://www.apple.com/ Randall Gellens QUALCOMM Incorporated 5775 Morehouse Dr. San Diego, CA 92121-2779 USA EMail: randy@qualcomm.com Daboo & Gellens Experimental [Page 30] RFC 5257 IMAP ANNOTATE Extension June 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Daboo & Gellens Experimental [Page 31] ================================================ FILE: docs/rfcs/rfc5258.IMAP4_LIST_command_extension.txt ================================================ Network Working Group B. Leiba Request for Comments: 5258 IBM T.J. Watson Research Center Obsoletes: 3348 A. Melnikov Updates: 2193 Isode Limited Category: Standards Track June 2008 Internet Message Access Protocol version 4 - LIST Command Extensions Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract IMAP4 has two commands for listing mailboxes: LIST and LSUB. As we have added extensions, such as Mailbox Referrals, that have required specialized lists we have had to expand the number of list commands, since each extension must add its function to both LIST and LSUB, and these commands are not, as they are defined, extensible. If we've needed the extensions to work together, we've had to add a set of commands to mix the different options, the set increasing in size with each new extension. This document describes an extension to the base LIST command that will allow these additions to be done with mutually compatible options to the LIST command, avoiding the exponential increase in specialized list commands. Leiba & Melnikov Standards Track [Page 1] RFC 5258 IMAP4 LIST Command Extensions June 2008 Table of Contents 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 2. Conventions Used in This Document . . . . . . . . . . . . . . 4 3. Extended LIST Command . . . . . . . . . . . . . . . . . . . . 4 3.1. Initial List of Selection Options . . . . . . . . . . . . 7 3.2. Initial List of Return Options . . . . . . . . . . . . . . 8 3.3. General Principles for Returning LIST Responses . . . . . 9 3.4. Additional Requirements on LIST-EXTENDED Clients . . . . . 9 3.5. CHILDINFO Extended Data Item . . . . . . . . . . . . . . . 10 4. The CHILDREN Return Option . . . . . . . . . . . . . . . . . . 11 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 12 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 19 7. Internationalization Considerations . . . . . . . . . . . . . 22 8. Security Considerations . . . . . . . . . . . . . . . . . . . 23 9. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 23 9.1. Guidelines for IANA . . . . . . . . . . . . . . . . . . . 23 9.2. Registration Procedure and Change Control . . . . . . . . 23 9.3. Registration Template for LIST-EXTENDED Options . . . . . 25 9.4. Initial LIST-EXTENDED Option Registrations . . . . . . . . 25 9.5. Registration Template for LIST-EXTENDED Extended Data Item . . . . . . . . . . . . . . . . . . . . . . . . . . . 28 9.6. Initial LIST-EXTENDED Extended Data Item Registrations . . 28 10. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 29 11. References . . . . . . . . . . . . . . . . . . . . . . . . . . 29 11.1. Normative References . . . . . . . . . . . . . . . . . . . 29 11.2. Informative References . . . . . . . . . . . . . . . . . . 30 Leiba & Melnikov Standards Track [Page 2] RFC 5258 IMAP4 LIST Command Extensions June 2008 1. Introduction and Overview The LIST command is extended by amending the syntax to allow options and multiple patterns to be specified. The list of options replaces the several commands that are currently used to mix and match the information requested. The new syntax is backward compatible, with no ambiguity: the new syntax is being used if one of the following conditions is true: 1. if the first word after the command name begins with a parenthesis ("LIST selection options") 2. if the second word after the command name begins with a parenthesis ("multiple mailbox patterns") 3. if the LIST command has more than 2 parameters ("LIST return options") Otherwise the original syntax is used. By adding options to the LIST command, we are announcing the intent to phase out and eventually to deprecate the RLIST and RLSUB commands described in [MBRef]. We are also defining the mechanism to request extended mailbox information, such as is described in the Child Mailbox Extension [CMbox]. The base LSUB command is not deprecated by this extension; rather, this extension adds a way to obtain subscription information with more options, with those server implementations that support it. Clients that simply need a list of subscribed mailboxes, as provided by the LSUB command, SHOULD continue to use that command. This document defines an IMAP4 extension that is identified by the capability string "LIST-EXTENDED". The LIST-EXTENDED extension makes the following changes to the IMAP4 protocol, which are described in more detail in Section 3 and Section 4: a. defines new syntax for LIST command options. b. extends LIST to allow for multiple mailbox patterns. c. adds LIST command selection options: SUBSCRIBED, REMOTE, and RECURSIVEMATCH. d. adds LIST command return options: SUBSCRIBED and CHILDREN. e. adds new mailbox attributes: "\NonExistent", "\Subscribed", "\Remote", "\HasChildren", and "\HasNoChildren". Leiba & Melnikov Standards Track [Page 3] RFC 5258 IMAP4 LIST Command Extensions June 2008 f. adds CHILDINFO extended data item. 2. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" are used in this document as specified in RFC 2119 [Kwds]. The term "canonical LIST pattern" refers to the canonical pattern constructed internally by the server from the reference and mailbox name arguments (Section 6.3.8 of [IMAP4]). The [IMAP4] LIST command returns only mailboxes that match the canonical LIST pattern. Other terms are introduced where they are referenced for the first time. 3. Extended LIST Command This extension updates the syntax of the LIST command to allow for multiple mailbox patterns to be specified, if they are enclosed in parentheses. A mailbox name matches a list of mailbox patterns if it matches at least one mailbox pattern. If a mailbox name matches multiple mailbox patterns from the list, the server SHOULD return only a single LIST response. Note that the non-extended LIST command is required to treat an empty ("" string) mailbox name argument as a special request to return the hierarchy delimiter and the root name of the name given in the reference parameter (as per [IMAP4]). However, ANY extended LIST command (extended in any of 3 ways specified in Section 1, or any combination thereof) MUST NOT treat the empty mailbox name as such a special request, and any regular processing described in this document applies. In particular, if an extended LIST command has multiple mailbox names and one (or more) of them is the empty string, the empty string MUST be ignored for the purpose of matching. Some servers might restrict which patterns are allowed in a LIST command. If a server doesn't accept a particular pattern, it MUST silently ignore it. The LIST command syntax is also extended in two additional ways: by adding a parenthesized list of command options between the command name and the reference name (LIST selection options) and an optional Leiba & Melnikov Standards Track [Page 4] RFC 5258 IMAP4 LIST Command Extensions June 2008 list of options at the end that control what kind of information should be returned (LIST return options). See the formal syntax in Section 6 for specific details. A LIST selection option tells the server which mailbox names should be selected by the LIST operation. The server should return information about all mailbox names that match any of the "canonical LIST pattern" (as described above) and satisfy additional selection criteria (if any) specified by the LIST selection options. Let's call any such mailbox name a "matched mailbox name". When multiple selection options are specified, the server MUST return information about mailbox names that satisfy every selection option, unless a description of a particular specified option prescribes special rules. An example of an option prescribing special rules is the RECURSIVEMATCH selection option described later in this section. We will use the term "selection criteria" when referring collectively to all selection options specified in a LIST command. A LIST return option controls which information is returned for each matched mailbox name. Note that return options MUST NOT cause the server to report information about additional mailbox names. If the client has not specified any return option, only information about attributes should be returned by the server. (Of course, the server is allowed to include any other information at will.) Both selection and return command options will be defined in this document and in approved extension documents; each option will be enabled by a capability string (one capability may enable multiple options), and a client MUST NOT send an option for which the server has not advertised support. A server MUST respond to options it does not recognize with a BAD response. The client SHOULD NOT specify any option more than once; however, if the client does this, the server MUST act as if it received the option only once. The order in which options are specified by the client is not significant. In general, each selection option except RECURSIVEMATCH will have a corresponding return option. The REMOTE selection option is an anomaly in this regard, and does not have a corresponding return option. That is because it expands, rather than restricts, the set of mailboxes that are returned. Future extensions to this specification should keep parallelism in mind and define a pair of corresponding options. This extension is identified by the capability string "LIST-EXTENDED", and support for it is a prerequisite for any future extensions that require specialized forms of the LIST command. Such extensions MUST refer to this document and MUST add their function through command options as described herein. Note that extensions Leiba & Melnikov Standards Track [Page 5] RFC 5258 IMAP4 LIST Command Extensions June 2008 that don't require support for an extended LIST command, but use extended LIST responses (see below), don't need to advertise the "LIST-EXTENDED" capability string. This extension also defines extensions to the LIST response, allowing a series of extended fields at the end, a parenthesized list of tagged data (also referred to as "extended data item"). The first element of an extended field is a tag, which identifies the type of data. Tags MUST be registered with IANA, as described in Section 9.5 of this document. An example of such an extended set might be tablecloth (("edge" "lacy") ("color" "red"))) (X-Sample "text")) or tablecloth ("edge" "lacy")) (X-Sample "text" "more text")) See the formal syntax, in Section 6, for the full syntactic details. The server MUST NOT return any extended data item unless the client has expressed its ability to support extended LIST responses, for example, by using an extended LIST command. The server MAY return data in the extended fields that was not directly solicited by the client in the corresponding LIST command. For example, the client can enable extra extended fields by using another IMAP extension that make use of the extended LIST responses. The client MUST ignore all extended fields it doesn't recognize. The LIST-EXTENDED capability also defines several new mailbox attributes. The "\NonExistent" attribute indicates that a mailbox name does not refer to an existing mailbox. Note that this attribute is not meaningful by itself, as mailbox names that match the canonical LIST pattern but don't exist must not be returned unless one of the two conditions listed below is also satisfied: a. The mailbox name also satisfies the selection criteria (for example, it is subscribed and the "SUBSCRIBED" selection option has been specified). b. "RECURSIVEMATCH" has been specified, and the mailbox name has at least one descendant mailbox name that does not match the LIST pattern and does match the selection criteria. In practice, this means that the "\NonExistent" attribute is usually returned with one or more of "\Subscribed", "\Remote", "\HasChildren", or the CHILDINFO extended data item (see their description below). Leiba & Melnikov Standards Track [Page 6] RFC 5258 IMAP4 LIST Command Extensions June 2008 The "\NonExistent" attribute implies "\NoSelect". The "\NonExistent" attribute MUST be supported and MUST be accurately computed. 3.1. Initial List of Selection Options The selection options defined in this specification are as follows: SUBSCRIBED - causes the LIST command to list subscribed names, rather than the existing mailboxes. This will often be a subset of the actual mailboxes. It's also possible for this list to contain the names of mailboxes that don't exist. In any case, the list MUST include exactly those mailbox names that match the canonical list pattern and are subscribed to. This option is intended to supplement the LSUB command. Of particular note are the mailbox attributes as returned by this option, compared with what is returned by LSUB. With the latter, the attributes returned may not reflect the actual attribute status on the mailbox name, and the \NoSelect attribute has a second special meaning (it indicates that this mailbox is not, itself, subscribed, but that it has descendant mailboxes that are). With the SUBSCRIBED selection option described here, the attributes are accurate and complete, and have no special meanings. "LSUB" and "LIST (SUBSCRIBED)" are, thus, not the same thing, and some servers must do significant extra work to respond to "LIST (SUBSCRIBED)". Because of this, clients SHOULD continue to use "LSUB" unless they specifically want the additional information offered by "LIST (SUBSCRIBED)". This option defines a new mailbox attribute, "\Subscribed", that indicates that a mailbox name is subscribed to. The "\Subscribed" attribute MUST be supported and MUST be accurately computed when the SUBSCRIBED selection option is specified. Note that the SUBSCRIBED selection option implies the SUBSCRIBED return option (see below). REMOTE - causes the LIST command to show remote mailboxes as well as local ones, as described in [MBRef]. This option is intended to replace the RLIST command and, in conjunction with the SUBSCRIBED selection option, the RLSUB command. This option defines a new mailbox attribute, "\Remote", that indicates that a mailbox is a remote mailbox. The "\Remote" attribute MUST be accurately computed when the REMOTE option is specified. The REMOTE selection option has no interaction with other options. Its effect is to tell the server to apply the other options, if Leiba & Melnikov Standards Track [Page 7] RFC 5258 IMAP4 LIST Command Extensions June 2008 any, to remote mailboxes, in addition to local ones. In particular, it has no interaction with RECURSIVEMATCH (see below). A request for (REMOTE RECURSIVEMATCH) is invalid, because a request for (RECURSIVEMATCH) is. A request for (REMOTE RECURSIVEMATCH SUBSCRIBED) is asking for all subscribed mailboxes, both local and remote. RECURSIVEMATCH - this option forces the server to return information about parent mailboxes that don't match other selection options, but have some submailboxes that do. Information about children is returned in the CHILDINFO extended data item, as described in Section 3.5. Note 1: In order for a parent mailbox to be returned, it still has to match the canonical LIST pattern. Note 2: When returning the CHILDINFO extended data item, it doesn't matter whether or not the submailbox matches the canonical LIST pattern. See also example 9 in Section 5. The RECURSIVEMATCH option MUST NOT occur as the only selection option (or only with REMOTE), as it only makes sense when other selection options are also used. The server MUST return BAD tagged response in such case. Note that even if the RECURSIVEMATCH option is specified, the client MUST still be able to handle a case when a CHILDINFO extended data item is returned and there are no submailboxes that meet the selection criteria of the subsequent LIST command, as they can be deleted/renamed after the LIST response was sent, but before the client had a chance to access them. 3.2. Initial List of Return Options The return options defined in this specification are as follows: SUBSCRIBED - causes the LIST command to return subscription state for all matching mailbox names. The "\Subscribed" attribute MUST be supported and MUST be accurately computed when the SUBSCRIBED return option is specified. Further, all mailbox flags MUST be accurately computed (this differs from the behavior of the LSUB command). CHILDREN - requests mailbox child information as originally proposed in [CMbox]. See Section 4, below, for details. This option MUST be supported by all servers. Leiba & Melnikov Standards Track [Page 8] RFC 5258 IMAP4 LIST Command Extensions June 2008 3.3. General Principles for Returning LIST Responses This section outlines several principles that can be used by server implementations of this document to decide whether a LIST response should be returned, as well as how many responses and what kind of information they may contain. 1. At most one LIST response should be returned for each mailbox name that matches the canonical LIST pattern. Server implementors must not assume that clients will be able to assemble mailbox attributes and other information returned in multiple LIST responses. 2. There are only two reasons for including a matching mailbox name in the responses to the LIST command (note that the server is allowed to return unsolicited responses at any time, and such responses are not governed by this rule): A. The mailbox name also satisfies the selection criteria. B. The mailbox name doesn't satisfy the selection criteria, but it has at least one descendant mailbox name that satisfies the selection criteria and that doesn't match the canonical LIST pattern. For more information on this case, see the CHILDINFO extended data item described in Section 3.5. Note that the CHILDINFO extended data item can only be returned when the RECURSIVEMATCH selection option is specified. 3. Attributes returned in the same LIST response must be treated additively. For example, the following response S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" means that the "Fruit/Peach" mailbox doesn't exist, but it is subscribed. 3.4. Additional Requirements on LIST-EXTENDED Clients All clients that support this extension MUST treat an attribute with a stronger meaning as implying any attribute that can be inferred from it. For example, the client must treat the presence of the \NoInferiors attribute as if the \HasNoChildren attribute was also sent by the server. Leiba & Melnikov Standards Track [Page 9] RFC 5258 IMAP4 LIST Command Extensions June 2008 The following table summarizes inference rules described in Section 3. +--------------------+-------------------+ | returned attribute | implied attribute | +--------------------+-------------------+ | \NoInferiors | \HasNoChildren | | \NonExistent | \NoSelect | +--------------------+-------------------+ 3.5. CHILDINFO Extended Data Item The CHILDINFO extended data item MUST NOT be returned unless the client has specified the RECURSIVEMATCH selection option. The CHILDINFO extended data item in a LIST response describes the selection criteria that has caused it to be returned and indicates that the mailbox has at least one descendant mailbox that matches the selection criteria. The LSUB command indicates this condition by using the "\NoSelect" attribute, but the LIST (SUBSCRIBED) command MUST NOT do that, since "\NoSelect" retains its original meaning here. Further, the CHILDINFO extended data item is more general, in that it can be used with any extended set of selection criteria. Note: Some servers allow for mailboxes to exist without requiring their parent to exist. For example, a mailbox "Customers/ABC" can exist while the mailbox "Customers" does not. As CHILDINFO extended data item is not allowed if the RECURSIVEMATCH selection option is not specified, such servers SHOULD use the "\NonExistent \HasChildren" attribute pair to signal to the client that there is a descendant mailbox that matches the selection criteria. See example 11 in Section 5. The returned selection criteria allow the client to distinguish a solicited response from an unsolicited one, as well as to distinguish among solicited responses caused by multiple pipelined LIST commands that specify different criteria. Servers SHOULD ONLY return a non-matching mailbox name along with CHILDINFO if at least one matching child is not also being returned. That is, servers SHOULD suppress redundant CHILDINFO responses. Examples 8 and 10 in Section 5 demonstrate the difference between present CHILDINFO extended data item and the "\HasChildren" attribute. Leiba & Melnikov Standards Track [Page 10] RFC 5258 IMAP4 LIST Command Extensions June 2008 The following table summarizes interaction between the "\NonExistent" attribute and CHILDINFO (the first column indicates whether the parent mailbox exists): +--------+--------------+--------------------+----------------------+ | exists | meets the | has a child that | returned | | | selection | meets the | LIST-EXTENDED | | | criteria | selection criteria | attributes and | | | | | CHILDINFO | +--------+--------------+--------------------+----------------------+ | no | no | no | no LIST response | | | | | returned | | yes | no | no | no LIST response | | | | | returned | | no | yes | no | (\NonExistent | | | | | ) | | yes | yes | no | () | | no | no | yes | (\NonExistent) + | | | | | CHILDINFO | | yes | no | yes | () + CHILDINFO | | no | yes | yes | (\NonExistent | | | | | ) + CHILDINFO | | yes | yes | yes | () + CHILDINFO | +--------+--------------+--------------------+----------------------+ where is one or more attributes that correspond to the selection criteria; for example, for the SUBSCRIBED option the is \Subscribed. 4. The CHILDREN Return Option The CHILDREN return option implements the Child Mailbox Extension, originally proposed by Mike Gahrns and Raymond Cheng, of Microsoft Corporation. Most of the information in this section is taken directly from their original specification [CMbox]. The CHILDREN return option is simply an indication that the client wants this information; a server MAY provide it even if the option is not specified. Many IMAP4 [IMAP4] clients present to the user a hierarchical view of the mailboxes that a user has access to. Rather than initially presenting to the user the entire mailbox hierarchy, it is often preferable to show to the user a collapsed outline list of the mailbox hierarchy (particularly if there is a large number of mailboxes). The user can then expand the collapsed outline hierarchy as needed. It is common to include within the collapsed hierarchy a visual clue (such as a ''+'') to indicate that there are child mailboxes under a particular mailbox. When the visual clue is Leiba & Melnikov Standards Track [Page 11] RFC 5258 IMAP4 LIST Command Extensions June 2008 clicked, the hierarchy list is expanded to show the child mailboxes. The CHILDREN return option provides a mechanism for a client to efficiently determine whether a particular mailbox has children, without issuing a LIST "" * or a LIST "" % for each mailbox name. The CHILDREN return option defines two new attributes that MUST be returned within a LIST response: \HasChildren and \HasNoChildren. Although these attributes MAY be returned in response to any LIST command, the CHILDREN return option is provided to indicate that the client particularly wants this information. If the CHILDREN return option is present, the server MUST return these attributes even if their computation is expensive. \HasChildren The presence of this attribute indicates that the mailbox has child mailboxes. A server SHOULD NOT set this attribute if there are child mailboxes and the user does not have permission to access any of them. In this case, \HasNoChildren SHOULD be used. In many cases, however, a server may not be able to efficiently compute whether a user has access to any child mailbox. Note that even though the \HasChildren attribute for a mailbox must be correct at the time of processing of the mailbox, a client must be prepared to deal with a situation when a mailbox is marked with the \HasChildren attribute, but no child mailbox appears in the response to the LIST command. This might happen, for example, due to children mailboxes being deleted or made inaccessible to the user (using access control) by another client before the server is able to list them. \HasNoChildren The presence of this attribute indicates that the mailbox has NO child mailboxes that are accessible to the currently authenticated user. It is an error for the server to return both a \HasChildren and a \HasNoChildren attribute in the same LIST response. Note: the \HasNoChildren attribute should not be confused with the IMAP4 [IMAP4] defined attribute \NoInferiors, which indicates that no child mailboxes exist now and none can be created in the future. Leiba & Melnikov Standards Track [Page 12] RFC 5258 IMAP4 LIST Command Extensions June 2008 5. Examples 1: The first example shows the complete local hierarchy that will be used for the other examples. C: A01 LIST "" "*" S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST () "/" "Fruit" S: * LIST () "/" "Fruit/Apple" S: * LIST () "/" "Fruit/Banana" S: * LIST () "/" "Tofu" S: * LIST () "/" "Vegetable" S: * LIST () "/" "Vegetable/Broccoli" S: * LIST () "/" "Vegetable/Corn" S: A01 OK done 2: In the next example, we will see the subscribed mailboxes. This is similar to, but not equivalent with, . Note that the mailbox called "Fruit/Peach" is subscribed to, but does not actually exist (perhaps it was deleted while still subscribed). The "Fruit" mailbox is not subscribed to, but it has two subscribed children. The "Vegetable" mailbox is subscribed and has two children; one of them is subscribed as well. C: A02 LIST (SUBSCRIBED) "" "*" S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" S: * LIST (\Subscribed) "/" "Fruit/Banana" S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" S: * LIST (\Subscribed) "/" "Vegetable" S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" S: A02 OK done 3: The next example shows the use of the CHILDREN option. The client, without having to list the second level of hierarchy, now knows which of the top-level mailboxes have submailboxes (children) and which do not. Note that it's not necessary for the server to return the \HasNoChildren attribute for the inbox, because the \NoInferiors attribute already implies that, and has a stronger meaning. C: A03 LIST () "" "%" RETURN (CHILDREN) S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST (\HasChildren) "/" "Fruit" S: * LIST (\HasNoChildren) "/" "Tofu" S: * LIST (\HasChildren) "/" "Vegetable" S: A03 OK done Leiba & Melnikov Standards Track [Page 13] RFC 5258 IMAP4 LIST Command Extensions June 2008 4: In this example, we see more mailboxes that reside on another server. This is similar to the command . C: A04 LIST (REMOTE) "" "%" RETURN (CHILDREN) S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST (\HasChildren) "/" "Fruit" S: * LIST (\HasNoChildren) "/" "Tofu" S: * LIST (\HasChildren) "/" "Vegetable" S: * LIST (\Remote) "/" "Bread" S: * LIST (\HasChildren \Remote) "/" "Meat" S: A04 OK done 5: The following example also requests the server to include mailboxes that reside on another server. The server returns information about all mailboxes that are subscribed. This is similar to the command . We also see the use of two selection options. C: A05 LIST (REMOTE SUBSCRIBED) "" "*" S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" S: * LIST (\Subscribed) "/" "Fruit/Banana" S: * LIST (\Subscribed \NonExistent) "/" "Fruit/Peach" S: * LIST (\Subscribed) "/" "Vegetable" S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" S: * LIST (\Remote \Subscribed) "/" "Bread" S: A05 OK done 6: The following example requests the server to include mailboxes that reside on another server. The server is asked to return subscription information for all returned mailboxes. This is different from the example above. Note that the output of this command is not a superset of the output in the previous example, as it doesn't include LIST response for the non-existent "Fruit/Peach". C: A06 LIST (REMOTE) "" "*" RETURN (SUBSCRIBED) S: * LIST (\Marked \NoInferiors \Subscribed) "/" "inbox" S: * LIST () "/" "Fruit" S: * LIST () "/" "Fruit/Apple" S: * LIST (\Subscribed) "/" "Fruit/Banana" S: * LIST () "/" "Tofu" S: * LIST (\Subscribed) "/" "Vegetable" S: * LIST (\Subscribed) "/" "Vegetable/Broccoli" S: * LIST () "/" "Vegetable/Corn" S: * LIST (\Remote \Subscribed) "/" "Bread" S: * LIST (\Remote) "/" "Meat" S: A06 OK done Leiba & Melnikov Standards Track [Page 14] RFC 5258 IMAP4 LIST Command Extensions June 2008 7: In the following example, the client has specified multiple mailbox patterns. Note that this example does not use the mailbox hierarchy used in the previous examples. C: BBB LIST "" ("INBOX" "Drafts" "Sent/%") S: * LIST () "/" "INBOX" S: * LIST (\NoInferiors) "/" "Drafts" S: * LIST () "/" "Sent/March2004" S: * LIST (\Marked) "/" "Sent/December2003" S: * LIST () "/" "Sent/August2004" S: BBB OK done 8: The following example demonstrates the difference between the \HasChildren attribute and the CHILDINFO extended data item. Let's assume there is the following hierarchy: C: C01 LIST "" "*" S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST () "/" "Foo" S: * LIST () "/" "Foo/Bar" S: * LIST () "/" "Foo/Baz" S: * LIST () "/" "Moo" S: C01 OK done If the client asks RETURN (CHILDREN), it will get this: C: CA3 LIST "" "%" RETURN (CHILDREN) S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST (\HasChildren) "/" "Foo" S: * LIST (\HasNoChildren) "/" "Moo" S: CA3 OK done A) Let's also assume that the mailbox "Foo/Baz" is the only subscribed mailbox. Then we get this result: C: C02 LIST (SUBSCRIBED) "" "*" S: * LIST (\Subscribed) "/" "Foo/Baz" S: C02 OK done Now, if the client issues , the server will return no mailboxes (as the mailboxes "Moo", "Foo", and "Inbox" are NOT subscribed). However, if the client issues this: C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" S: * LIST () "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) S: C04 OK done Leiba & Melnikov Standards Track [Page 15] RFC 5258 IMAP4 LIST Command Extensions June 2008 (i.e., the mailbox "Foo" is not subscribed, but it has a child that is.) A1) If the mailbox "Foo" had also been subscribed, the last command would return this: C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" S: * LIST (\Subscribed) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) S: C04 OK done or even this: C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" S: * LIST (\Subscribed \HasChildren) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) S: C04 OK done A2) If we assume instead that the mailbox "Foo" is not part of the original hierarchy and is not subscribed, the last command will give this result: C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" S: * LIST (\NonExistent) "/" "Foo" ("CHILDINFO" ("SUBSCRIBED")) S: C04 OK done B) Now, let's assume that no mailbox is subscribed. In this case, the command will return no responses, as there are no subscribed children (even though "Foo" has children). C) And finally, suppose that only the mailboxes "Foo" and "Moo" are subscribed. In that case, we see this result: C: C04 LIST (SUBSCRIBED RECURSIVEMATCH) "" "%" RETURN (CHILDREN) S: * LIST (\HasChildren \Subscribed) "/" "Foo" S: * LIST (\HasNoChildren \Subscribed) "/" "Moo" S: C04 OK done (which means that the mailbox "Foo" has children, but none of them is subscribed). 9: The following example demonstrates that the CHILDINFO extended data item is returned whether or not children mailboxes match the canonical LIST pattern. Leiba & Melnikov Standards Track [Page 16] RFC 5258 IMAP4 LIST Command Extensions June 2008 Let's assume there is the following hierarchy: C: D01 LIST "" "*" S: * LIST (\Marked \NoInferiors) "/" "inbox" S: * LIST () "/" "foo2" S: * LIST () "/" "foo2/bar1" S: * LIST () "/" "foo2/bar2" S: * LIST () "/" "baz2" S: * LIST () "/" "baz2/bar2" S: * LIST () "/" "baz2/bar22" S: * LIST () "/" "baz2/bar222" S: * LIST () "/" "eps2" S: * LIST () "/" "eps2/mamba" S: * LIST () "/" "qux2/bar2" S: D01 OK done And that the following mailboxes are subscribed: C: D02 LIST (SUBSCRIBED) "" "*" S: * LIST (\Subscribed) "/" "foo2/bar1" S: * LIST (\Subscribed) "/" "foo2/bar2" S: * LIST (\Subscribed) "/" "baz2/bar2" S: * LIST (\Subscribed) "/" "baz2/bar22" S: * LIST (\Subscribed) "/" "baz2/bar222" S: * LIST (\Subscribed) "/" "eps2" S: * LIST (\Subscribed) "/" "eps2/mamba" S: * LIST (\Subscribed) "/" "qux2/bar2" S: D02 OK done The client issues the following command first: C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*2" S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\Subscribed) "/" "foo2/bar2" S: * LIST (\Subscribed) "/" "baz2/bar2" S: * LIST (\Subscribed) "/" "baz2/bar22" S: * LIST (\Subscribed) "/" "baz2/bar222" S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\Subscribed) "/" "qux2/bar2" S: D03 OK done and the server may also include (but this would violate a SHOULD NOT in Section 3.5, because CHILDINFO is redundant) S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\NonExistent) "/" "qux2" ("CHILDINFO" ("SUBSCRIBED")) Leiba & Melnikov Standards Track [Page 17] RFC 5258 IMAP4 LIST Command Extensions June 2008 The CHILDINFO extended data item is returned for mailboxes "foo2", "baz2", and "eps2", because all of them have subscribed children, even though for the mailbox "foo2" only one of the two subscribed children matches the pattern, for the mailbox "baz2" all the subscribed children match the pattern, and for the mailbox "eps2" none of the subscribed children matches the pattern. Note that if the client issues C: D03 LIST (RECURSIVEMATCH SUBSCRIBED) "" "*" S: * LIST () "/" "foo2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\Subscribed) "/" "foo2/bar1" S: * LIST (\Subscribed) "/" "foo2/bar2" S: * LIST () "/" "baz2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\Subscribed) "/" "baz2/bar2" S: * LIST (\Subscribed) "/" "baz2/bar22" S: * LIST (\Subscribed) "/" "baz2/bar222" S: * LIST (\Subscribed) "/" "eps2" ("CHILDINFO" ("SUBSCRIBED")) S: * LIST (\Subscribed) "/" "eps2/mamba" S: * LIST (\Subscribed) "/" "qux2/bar2" S: D03 OK done The LIST responses for mailboxes "foo2", "baz2", and "eps2" still have the CHILDINFO extended data item, even though this information is redundant and the client can determine it by itself. 10: The following example shows usage of multiple mailbox patterns. It also demonstrates that the presence of the CHILDINFO extended data item doesn't necessarily imply \HasChildren. C: a1 LIST "" ("foo" "foo/*") S: * LIST () "/" foo S: a1 OK done C: a2 LIST (SUBSCRIBED) "" "foo/*" S: * LIST (\Subscribed \NonExistent) "/" foo/bar S: a2 OK done C: a3 LIST (SUBSCRIBED RECURSIVEMATCH) "" foo RETURN (CHILDREN) S: * LIST (\HasNoChildren) "/" foo ("CHILDINFO" ("SUBSCRIBED")) S: a3 OK done 11: The following example shows how a server that supports missing mailbox hierarchy elements can signal to a client that didn't specify the RECURSIVEMATCH selection option that there is a child mailbox that matches the selection criteria. Leiba & Melnikov Standards Track [Page 18] RFC 5258 IMAP4 LIST Command Extensions June 2008 C: a1 LIST (REMOTE) "" * S: * LIST () "/" music/rock S: * LIST (\Remote) "/" also/jazz S: a1 OK done C: a2 LIST () "" % S: * LIST (\NonExistent \HasChildren) "/" music S: a2 OK done C: a3 LIST (REMOTE) "" % S: * LIST (\NonExistent \HasChildren) "/" music S: * LIST (\NonExistent \HasChildren) "/" also S: a3 OK done C: a3.1 LIST "" (% music/rock) S: * LIST () "/" music/rock S: a3.1 OK done Because "music/rock" is the only mailbox under "music", there's no need for the server to also return "music". However clients must handle both cases. 6. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) as described in [ABNF]. Terms not defined here are taken from [IMAP4]. In particular, note that the version of "mailbox-list" below, which defines the payload of the LIST response, updates the version defined in the IMAP specification. It is pointed to by "mailbox-data", which is defined in [IMAP4]. "vendor-token" is defined in [ACAP]. Note that this normative reference to ACAP will be an issue in moving this spec forward, since it introduces a dependency on ACAP. The definitions of "vendor-token" and of the IANA registry must eventually go somewhere else, in a document that can be moved forward on the standards track independently of ACAP. Leiba & Melnikov Standards Track [Page 19] RFC 5258 IMAP4 LIST Command Extensions June 2008 childinfo-extended-item = "CHILDINFO" SP "(" list-select-base-opt-quoted *(SP list-select-base-opt-quoted) ")" ; Extended data item (mbox-list-extended-item) ; returned when the RECURSIVEMATCH ; selection option is specified. ; Note 1: the CHILDINFO tag can be returned ; with and without surrounding quotes, as per ; mbox-list-extended-item-tag production. ; Note 2: The selection options are always returned ; quoted, unlike their specification in ; the extended LIST command. child-mbox-flag = "\HasChildren" / "\HasNoChildren" ; attributes for CHILDREN return option, at most one ; possible per LIST response eitem-standard-tag = atom ; a tag for extended list data defined in a Standard ; Track or Experimental RFC. eitem-vendor-tag = vendor-token "-" atom ; a vendor-specific tag for extended list data list = "LIST" [SP list-select-opts] SP mailbox SP mbox-or-pat [SP list-return-opts] list-return-opts = "RETURN" SP "(" [return-option *(SP return-option)] ")" ; list return options, e.g., CHILDREN list-select-base-opt = "SUBSCRIBED" / option-extension ; options that can be used by themselves list-select-base-opt-quoted = DQUOTE list-select-base-opt DQUOTE list-select-independent-opt = "REMOTE" / option-extension ; options that do not syntactically interact with ; other options list-select-mod-opt = "RECURSIVEMATCH" / option-extension ; options that require a list-select-base-opt ; to also be present list-select-opt = list-select-base-opt / list-select-independent-opt / list-select-mod-opt ; An option registration template is described in ; Section 9.3 of this document. Leiba & Melnikov Standards Track [Page 20] RFC 5258 IMAP4 LIST Command Extensions June 2008 list-select-opts = "(" [ (*(list-select-opt SP) list-select-base-opt *(SP list-select-opt)) / (list-select-independent-opt *(SP list-select-independent-opt)) ] ")" ; Any number of options may be in any order. ; If a list-select-mod-opt appears, then a ; list-select-base-opt must also appear. ; This allows these: ; () ; (REMOTE) ; (SUBSCRIBED) ; (SUBSCRIBED REMOTE) ; (SUBSCRIBED RECURSIVEMATCH) ; (SUBSCRIBED REMOTE RECURSIVEMATCH) ; But does NOT allow these: ; (RECURSIVEMATCH) ; (REMOTE RECURSIVEMATCH) mailbox-list = "(" [mbx-list-flags] ")" SP (DQUOTE QUOTED-CHAR DQUOTE / nil) SP mailbox [SP mbox-list-extended] ; This is the list information pointed to by the ABNF ; item "mailbox-data", which is defined in [IMAP4] mbox-list-extended = "(" [mbox-list-extended-item *(SP mbox-list-extended-item)] ")" mbox-list-extended-item = mbox-list-extended-item-tag SP tagged-ext-val mbox-list-extended-item-tag = astring ; The content MUST conform to either "eitem-vendor-tag" ; or "eitem-standard-tag" ABNF productions. ; A tag registration template is described in this ; document in Section 9.5. mbx-list-oflag =/ child-mbox-flag / "\Subscribed" / "\Remote" mbx-list-sflag =/ "\NonExistent" mbox-or-pat = list-mailbox / patterns option-extension = (option-standard-tag / option-vendor-tag) [SP option-value] Leiba & Melnikov Standards Track [Page 21] RFC 5258 IMAP4 LIST Command Extensions June 2008 option-standard-tag = atom ; an option defined in a Standards Track or ; Experimental RFC option-val-comp = astring / option-val-comp *(SP option-val-comp) / "(" option-val-comp ")" option-value = "(" option-val-comp ")" option-vendor-tag = vendor-token "-" atom ; a vendor-specific option, non-standard patterns = "(" list-mailbox *(SP list-mailbox) ")" return-option = "SUBSCRIBED" / "CHILDREN" / option-extension tagged-ext-comp = astring / tagged-ext-comp *(SP tagged-ext-comp) / "(" tagged-ext-comp ")" ; Extensions that follow this general ; syntax should use nstring instead of ; astring when appropriate in the context ; of the extension. ; Note that a message set or a "number" ; can always be represented as an "atom". ; A URL should be represented as ; a "quoted" string. tagged-ext-simple = sequence-set / number tagged-ext-val = tagged-ext-simple / "(" [tagged-ext-comp] ")" 7. Internationalization Considerations The LIST command selection option types defined in this specification involve simple tests of mailbox properties. However, future extensions to LIST-EXTENDED may define selection options that do more sophisticated tests. In the case of a test that requires matching text, in the presence of the COMPARATOR [I18N] extension, the active comparator must be used to do comparisons. Such LIST-EXTENDED extensions MUST indicate in their specification the interaction with the COMPARATOR [I18N] extension. Leiba & Melnikov Standards Track [Page 22] RFC 5258 IMAP4 LIST Command Extensions June 2008 8. Security Considerations This document describes syntactic changes to the specification of the IMAP4 commands LIST, LSUB, RLIST, and RLSUB, and the modified LIST command has the same security considerations as those commands. They are described in [IMAP4] and [MBRef]. The Child Mailbox Extension provides a client a more efficient means of determining whether a particular mailbox has children. If a mailbox has children, but the currently authenticated user does not have access to any of them, the server SHOULD respond with a \HasNoChildren attribute. In many cases, however, a server may not be able to efficiently compute whether a user has access to any child mailbox. If such a server responds with a \HasChildren attribute, when in fact the currently authenticated user does not have access to any child mailboxes, potentially more information is conveyed about the mailbox than intended. In most situations, this will not be a security concern, because if information regarding whether a mailbox has children is considered sensitive, a user would not be granted access to that mailbox in the first place. The CHILDINFO extended data item has the same security considerations as the \HasChildren attribute described above. 9. IANA Considerations 9.1. Guidelines for IANA IANA has created two new registries for LIST-EXTENDED options and LIST-EXTENDED response data. The templates and the initial registrations are detailed below. 9.2. Registration Procedure and Change Control Registration of a LIST-EXTENDED option is done by filling in the template in Section 9.3 and sending it via electronic mail to iana@iana.org. Registration of a LIST-EXTENDED extended data item is done by filling in the template in Section 9.5 and sending it via electronic mail to iana@iana.org. IANA has the right to reject obviously bogus registrations, but will perform no review of claims made in the registration form. A LIST-EXTENDED option/extended data item name that starts with "V-" is reserved for vendor-specific options/extended data items. All options, whether they are vendor specific or global, should be registered with IANA. If a LIST-EXTENDED extended data item is returned as a result of requesting a particular LIST-EXTENDED option, Leiba & Melnikov Standards Track [Page 23] RFC 5258 IMAP4 LIST Command Extensions June 2008 the name of the option SHOULD be used as the name of the LIST-EXTENDED extended data item. Each vendor-specific option/extended data item MUST start with its vendor-token ("vendor prefix"). The vendor-token MUST be registered with IANA, using the [ACAP] vendor subtree registry. Standard LIST-EXTENDED option/extended data item names are case insensitive. If the vendor prefix is omitted from a vendor-specific LIST-EXTENDED option/extended data item name, the rest is case insensitive. The vendor prefix itself is not case sensitive, as it might contain non-ASCII characters. While the registration procedures do not require it, authors of LIST-EXTENDED options/extended data items are encouraged to seek community review and comment whenever that is feasible. Authors may seek community review by posting a specification of their proposed mechanism as an Internet-Draft. LIST-EXTENDED option/extended data items intended for widespread use should be standardized through the normal IETF process, when appropriate. Comments on registered LIST-EXTENDED options/extended response data should first be sent to the "owner" of the mechanism and/or to the IMAPEXT WG mailing list. Submitters of comments may, after a reasonable attempt to contact the owner, request IANA to attach their comment to the registration itself. If IANA approves of this, the comment will be made accessible in conjunction with the registration LIST-EXTENDED options/extended response data itself. Once a LIST-EXTENDED registration has been published by IANA, the author may request a change to its definition. The change request follows the same procedure as the registration request. The owner of a LIST-EXTENDED registration may pass responsibility for the registered option/extended data item to another person or agency by informing IANA; this can be done without discussion or review. The IESG may reassign responsibility for a LIST-EXTENDED option/extended data item. The most common case of this will be to enable changes to be made to mechanisms where the author of the registration has died, has moved out of contact, or is otherwise unable to make changes that are important to the community. LIST-EXTENDED registrations may not be deleted; mechanisms that are no longer believed appropriate for use can be declared OBSOLETE by a change to their "intended use" field. Such LIST-EXTENDED options/extended data items will be clearly marked in the lists published by IANA. Leiba & Melnikov Standards Track [Page 24] RFC 5258 IMAP4 LIST Command Extensions June 2008 The IESG is considered to be the owner of all LIST-EXTENDED options/extended data items that are on the IETF standards track. 9.3. Registration Template for LIST-EXTENDED Options To: iana@iana.org Subject: Registration of LIST-EXTENDED option X LIST-EXTENDED option name: LIST-EXTENDED option type: (One of SELECTION or RETURN) Implied return options(s), if the option type is SELECTION: (zero or more) LIST-EXTENDED option description: Published specification (optional, recommended): Security considerations: Intended usage: (One of COMMON, LIMITED USE, or OBSOLETE) Person and email address to contact for further information: Owner/Change controller: (Any other information that the author deems interesting may be added below this line.) 9.4. Initial LIST-EXTENDED Option Registrations The LIST-EXTENDED option registry has been populated with the following entries: 1. To: iana@iana.org Subject: Registration of LIST-EXTENDED option SUBSCRIBED LIST-EXTENDED option name: SUBSCRIBED LIST-EXTENDED option type: SELECTION Implied return options(s): SUBSCRIBED LIST-EXTENDED option description: Causes the LIST command to list subscribed mailboxes, rather than the actual mailboxes. Leiba & Melnikov Standards Track [Page 25] RFC 5258 IMAP4 LIST Command Extensions June 2008 Published specification: RFC 5258, Section 3. Security considerations: RFC 5258, Section 8. Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 2. To: iana@iana.org Subject: Registration of LIST-EXTENDED option REMOTE LIST-EXTENDED option name: REMOTE LIST-EXTENDED option type: SELECTION Implied return options(s): (none) LIST-EXTENDED option description: Causes the LIST command to return remote mailboxes as well as local ones, as described in RFC 2193. Published specification: RFC 5258, Section 3. Security considerations: RFC 5258, Section 8. Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 3. To: iana@iana.org Subject: Registration of LIST-EXTENDED option SUBSCRIBED LIST-EXTENDED option name: SUBSCRIBED LIST-EXTENDED option type: RETURN LIST-EXTENDED option description: Causes the LIST command to return subscription state. Published specification: RFC 5258, Section 3. Security considerations: RFC 5258, Section 8. Leiba & Melnikov Standards Track [Page 26] RFC 5258 IMAP4 LIST Command Extensions June 2008 Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 4. To: iana@iana.org Subject: Registration of LIST-EXTENDED option RECURSIVEMATCH LIST-EXTENDED option name: RECURSIVEMATCH LIST-EXTENDED option type: SELECTION Implied return options(s): (none) LIST-EXTENDED option description: Requests that CHILDINFO extended data item (childinfo-extended-item) is to be returned. Published specification: RFC 5258, Section 3. Security considerations: RFC 5258, Section 8. Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 5. To: iana@iana.org Subject: Registration of LIST-EXTENDED option CHILDREN LIST-EXTENDED option name: CHILDREN LIST-EXTENDED option type: RETURN LIST-EXTENDED option description: Requests mailbox child information. Published specification: RFC 5258, Section 3 and Section 4. Security considerations: RFC 5258, Section 8. Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Leiba & Melnikov Standards Track [Page 27] RFC 5258 IMAP4 LIST Command Extensions June 2008 Owner/Change controller: iesg@ietf.org 9.5. Registration Template for LIST-EXTENDED Extended Data Item To: iana@iana.org Subject: Registration of X LIST-EXTENDED extended data item LIST-EXTENDED extended data item tag: LIST-EXTENDED extended data item description: Which LIST-EXTENDED option(s) (and their types) causes this extended data item to be returned (if any): Published specification (optional, recommended): Security considerations: Intended usage: (One of COMMON, LIMITED USE, or OBSOLETE) Person and email address to contact for further information: Owner/Change controller: (Any other information that the author deems interesting may be added below this line.) 9.6. Initial LIST-EXTENDED Extended Data Item Registrations The LIST-EXTENDED extended data item registry has been populated with the following entries: 1. To: iana@iana.org Subject: Registration of CHILDINFO LIST-EXTENDED extended data item LIST-EXTENDED extended data item tag: CHILDINFO LIST-EXTENDED extended data item description: The CHILDINFO extended data item describes the selection criteria that has caused it to be returned and indicates that the mailbox has one or more child mailboxes that match the selection criteria. Which LIST-EXTENDED option(s) (and their types) causes this extended data item to be returned (if any): RECURSIVEMATCH selection option Leiba & Melnikov Standards Track [Page 28] RFC 5258 IMAP4 LIST Command Extensions June 2008 Published specification: RFC 5258, Section 3.5. Security considerations: RFC 5258, Section 8. Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 10. Acknowledgements Mike Gahrns and Raymond Cheng of Microsoft Corporation originally devised the Child Mailbox Extension and proposed it in 1997; the idea, as well as most of the text in Section 4, is theirs. This document is the result of discussions on the IMAP4 and IMAPEXT mailing lists and is meant to reflect consensus of those groups. In particular, Mark Crispin, Philip Guenther, Cyrus Daboo, Timo Sirainen, Ken Murchison, Rob Siemborski, Steve Hole, Arnt Gulbrandsen, Larry Greenfield, Dave Cridland, and Pete Maclean were active participants in those discussions or made suggestions to this document. 11. References 11.1. Normative References [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [ACAP] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [I18N] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet Message Access Protocol Internationalization", RFC 5255, June 2008. [IMAP4] Crispin, M., "Internet Message Access Protocol - Version 4rev1", RFC 3501, March 2003. [Kwds] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", RFC 2119, March 1997. [MBRef] Gahrns, M., "IMAP4 Mailbox Referrals", RFC 2193, September 1997. Leiba & Melnikov Standards Track [Page 29] RFC 5258 IMAP4 LIST Command Extensions June 2008 11.2. Informative References [CMbox] Gahrns, M. and R. Cheng, "The Internet Message Action Protocol (IMAP4) Child Mailbox Extension", RFC 3348, July 2002. Authors' Addresses Barry Leiba IBM T.J. Watson Research Center 19 Skyline Drive Hawthorne, NY 10532 US Phone: +1 914 784 7941 EMail: leiba@watson.ibm.com Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Leiba & Melnikov Standards Track [Page 30] RFC 5258 IMAP4 LIST Command Extensions June 2008 Full Copyright Statement Copyright (C) The IETF Trust (2008). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Leiba & Melnikov Standards Track [Page 31] ================================================ FILE: docs/rfcs/rfc5423.IM_Store_Events.txt ================================================ Network Working Group R. Gellens Request for Comments: 5423 QUALCOMM Inc. Category: Standards Track C. Newman Sun Microsystems March 2009 Internet Message Store Events Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (c) 2009 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents in effect on the date of publication of this document (http://trustee.ietf.org/license-info). Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Abstract One of the missing features in the existing Internet mail and messaging standards is a facility for server-to-server and server-to- client event notifications related to message store events. As the scope of Internet mail expands to support more diverse media (such as voice mail) and devices (such as cell phones) and to provide rich interactions with other services (such as web portals and legal compliance systems), the need for an interoperable notification system increases. This document attempts to enumerate the types of events that interest real-world consumers of such a system. This document describes events and event parameters that are useful for several cases, including notification to administrative systems and end users. This is not intended as a replacement for a message access facility such as IMAP. Gellens & Newman Standards Track [Page 1] RFC 5423 Internet Message Store Events March 2009 Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 2. Terminology . . . . . . . . . . . . . . . . . . . . . . . . . 3 3. Event Model . . . . . . . . . . . . . . . . . . . . . . . . . 4 4. Event Types . . . . . . . . . . . . . . . . . . . . . . . . . 5 4.1. Message Addition and Deletion . . . . . . . . . . . . . . 5 4.2. Message Flags . . . . . . . . . . . . . . . . . . . . . . 7 4.3. Access Accounting . . . . . . . . . . . . . . . . . . . . 8 4.4. Mailbox Management . . . . . . . . . . . . . . . . . . . . 8 5. Event Parameters . . . . . . . . . . . . . . . . . . . . . . . 10 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 14 7. Security Considerations . . . . . . . . . . . . . . . . . . . 14 8. Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . 15 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 15 9.1. Normative References . . . . . . . . . . . . . . . . . . . 15 9.2. Informative References . . . . . . . . . . . . . . . . . . 15 Appendix A. Future Extensions . . . . . . . . . . . . . . . . . . 17 1. Introduction A message store is used to organize Internet Messages [RFC5322] into one or more mailboxes (possibly hierarchical), annotate them in various ways, and provide access to these messages and associated metadata. Three different standards-based protocols have been widely deployed to remotely access a message store. The Post Office Protocol (POP) [RFC1939] provides simple download-and-delete access to a single mail drop (which is a subset of the functionality typically associated with a message store). The Internet Message Access Protocol (IMAP) [RFC3501] provides an extensible feature-rich model for online, offline, and disconnected access to a message store with minimal constraints on any associated "fat-client" user interface. Finally, mail access applications built on top of the Hypertext Transfer Protocol (HTTP) [RFC2616] that run in standards- based web browsers provide a third standards-based access mechanism for online-only access. While simple and/or ad-hoc mechanisms for notifications have sufficed to some degree in the past (e.g., "Simple New Mail Notification" [RFC4146], "IMAP4 IDLE Command" [RFC2177]), as the scope and importance of message stores expand, the demand for a more complete store notification system increases. Some of the driving forces behind this demand include: o Mobile devices with intermittent network connectivity that have "new mail" or "message count" indicators. Gellens & Newman Standards Track [Page 2] RFC 5423 Internet Message Store Events March 2009 o Unified messaging systems that include both Internet and voice mail require support for a message-waiting indicator on phones. o Interaction with systems for event-based or utility-computing billing. o Simplification of the process of passing message store events to non-Internet notification systems. o A calendar system may wish to subscribe to MessageNew notifications in order to support iMIP [RFC2447]. o Some jurisdictions have laws or regulations for information protection and auditing that require interoperable protocols between message stores built by messaging experts and compliance auditing systems built by compliance experts. Vendors who have deployed proprietary notification systems for their Internet message stores have seen significant demand to provide notifications for more and more events. As a first step towards building a notification system, this document attempts to enumerate the core events that real-world customers demand. This document includes those events that can be generated by the use of IMAP4rev1 [RFC3501] and some existing extensions. As new IMAP extensions are defined, or additional event types or parameters need to be added, the set specified here can be extended by means of an IANA registry with update requirements, as specified in Section 6. 1.1. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [RFC2119]. When these words appear in lower-case or with initial capital letters, they are not RFC 2119 key words. 2. Terminology The following terminology is used in this document: mailbox A container for Internet messages and/or child mailboxes. A mailbox may or may not permit delivery of new messages via a mail delivery agent. Gellens & Newman Standards Track [Page 3] RFC 5423 Internet Message Store Events March 2009 mailbox identifier A mailbox identifier provides sufficient information to identify a specific mailbox on a specific server instance. An IMAP URL can be a mailbox identifier. message access protocols Protocols that provide clients (e.g., a mail user agent or web browser) with access to the message store, including but not limited to IMAP, POP, and HTTP. message context As defined in [RFC3458]. UIDVALIDITY As defined in IMAP4rev1 [RFC3501]. UIDVALIDITY is critical to the correct operation of a caching mail client. When it changes, the client MUST flush its cache. It's particularly important to include UIDVALIDITY with event notifications related to message addition or removal in order to keep the message data correctly synchronized. 3. Event Model The events that are generated by a message store depend to some degree on the model used to represent a message store. The model the IETF has for a message store is implicit from IMAP4rev1 and extensions, so that model is assumed by this document. A message store event typically has an associated mailbox name and usually has an associated user name (or authorization identity if using the terminology from "Simple Authentication and Security Layer" (SASL) [RFC4422]). Events referring to a specific message can use an IMAP URL [RFC5092] to do so. Events referring to a set of messages can use an IMAP URL to the mailbox plus an IMAP UID (Unique Identifier) set. Each notification has a type and parameters. The type determines the type of event, while the parameters supply information about the context of the event that may be used to adjust subscription preferences or may simply supply data associated with the event. The types and parameter names in this document are restricted to US-ASCII printable characters, so these events can be easily mapped to an arbitrary notification system. However, this document assumes that arbitrary parameter values (including large and multi-line values) can be encoded with the notification system. Systems which lack that feature could only implement a subset of these events. Gellens & Newman Standards Track [Page 4] RFC 5423 Internet Message Store Events March 2009 This document does not indicate which event parameters are mandatory or optional. That is done in documents that specify specific message formats or bindings to a notification system. For scalability reasons, some degree of filtering at event generation is necessary. At the very least, the ability to turn on and off groups of related events and to suppress inclusion of large parameters (such as messageContent) is needed. A sophisticated publish/subscribe notification system may be able to propagate cumulative subscription information to the publisher. Some of these events might be logically collapsed into a single event type with a required parameter to distinguish between the cases (e.g., QuotaExceed and QuotaWithin). However, until such time that an event subscription model is formulated, it's not practical to make such decisions. We thus note only the fact that some of these events may be viewed as a single event type. 4. Event Types This section discusses the different types of events useful in a message store event notification system. The intention is to document the events sufficient to cover an overwhelming majority of known use cases while leaving less common event types for the future. This section mentions parameters that are important or specific to the events described here. Event parameters likely to be included in most or all notifications are discussed in the next section. 4.1. Message Addition and Deletion This section includes events related to message addition and deletion. MessageAppend A message was appended or concatenated to a mailbox by a message access client. For the most part, this is identical to the MessageNew event type except that the SMTP envelope information is not included as a parameter, but information about which protocol triggered the event MAY be included. See the MessageNew event for more information. MessageExpire One or more messages were expired from a mailbox due to server expiration policy and are no longer accessible by the end user. The parameters include a mailbox identifier that MUST include UIDVALIDITY and a UID set that describes the messages. Gellens & Newman Standards Track [Page 5] RFC 5423 Internet Message Store Events March 2009 Information about which server expiration policy was applied may be included in the future. MessageExpunge One or more messages were expunged from a mailbox by an IMAP CLOSE/EXPUNGE, POP3 DELE+QUIT, HTTP, or equivalent client action and are no longer accessible by the end user. The parameters include a mailbox identifier that MUST include UIDVALIDITY, a UID set, and MAY also indicate which access protocol triggered the event. MessageNew A new message was received into a mailbox via a message delivery agent. The parameters include a message identifier that, for IMAP- accessible message stores, MUST include UIDVALIDITY and a UID. The parameters MAY also include an SMTP envelope and other arbitrary message and mailbox metadata. In some cases, the entire new message itself may be included. The set of parameters SHOULD be adjustable to the client's preference, with limits set by server policy. An interesting policy, for example, would be to include messages up to 2K in size with the notification, but to include a URLAUTH [RFC4467] reference for larger messages. QuotaExceed An operation failed (typically MessageNew) because the user's mailbox exceeded one of the quotas (e.g., disk quota, message quota, quota by message context, etc.). The parameters SHOULD include at least the relevant user and quota and, optionally, the mailbox. Quota usage SHOULD be included if possible. Parameters needed to extend this to support quota by context are not presently described in this document but could be added in the future. QuotaWithin An operation occurred (typically MessageExpunge or MessageExpire) that reduced the user's quota usage under the limit. QuotaChange The user's quota was changed. Gellens & Newman Standards Track [Page 6] RFC 5423 Internet Message Store Events March 2009 4.2. Message Flags This section includes events related to changes in message flags. MessageRead One or more messages in the mailbox were marked as read or seen by a user. Note that POP has no concept of read or seen messages, so these events are only generated by IMAP or HTTP clients (or equivalent). The parameters include a mailbox identifier and a set of message UIDs. MessageTrash One or more messages were marked for future deletion by the user but are still accessible over the protocol (the user's client may or may not make these messages accessible through its user interface). The parameters include a mailbox identifier and a set of message UIDs. FlagsSet One or more messages in the mailbox had one or more IMAP flags or keywords set. The parameters include a list of IMAP flag or keyword names that were set, a mailbox identifier, and the set of UIDs of affected messages. The flagNames MUST NOT include \Recent. For compatibility with simpler clients, it SHOULD be configurable whether setting the \Seen or \Deleted flags results in this event or the simpler MessageRead/MessageTrash events. By default, the simpler message forms SHOULD be used for MessageRead and MessageTrash. FlagsClear One or more messages in the mailbox had one or more IMAP flags or keywords cleared. The parameters include a list of IMAP flag or keyword names that were cleared, a mailbox identifier, and the set of UIDs of affected messages. The flagNames parameter MUST NOT include \Recent. Gellens & Newman Standards Track [Page 7] RFC 5423 Internet Message Store Events March 2009 4.3. Access Accounting This section lists events related to message store access accounting. Login A user has logged into the system via IMAP, HTTP, POP, or some other mechanism. The parameters include the domain name and port used to access the server and the user's authorization identity. Additional possible parameters include the client's IP address and port, the authentication identity (if different from the authorization identity), the service name, the authentication mechanism, information about any negotiated security layers, a timestamp, and other information. Logout A user has logged out or otherwise been disconnected from the message store via IMAP, HTTP, POP, or some other mechanism. The parameters include the server domain name and the user's authorization identity. Additional parameters MAY include any of the information from the "Login" event as well as information about the type of disconnect (suggested values include graceful, abort, timeout, and security layer error), the duration of the connection or session, and other information. 4.4. Mailbox Management This section lists events related to the management of mailboxes. MailboxCreate A mailbox has been created, or an access control changed on an existing mailbox so that it is now accessible by the user. If the mailbox creation caused the creation of new mailboxes earlier in the hierarchy, separate MailboxCreate events are not generated, as their creation is implied. The parameters include the created mailbox identifier, its UIDVALIDITY for IMAP-accessible message stores, and MAY also indicate which access protocol triggered the event. Access and permissions information (such as Access Control List (ACL) [RFC4314] settings) require a standardized format to be included, and so are left for future extension. Gellens & Newman Standards Track [Page 8] RFC 5423 Internet Message Store Events March 2009 MailboxDelete A mailbox has been deleted, or an access control changed on an existing mailbox so that it is no longer accessible by the user. Note that if the mailbox has child mailboxes, only the specified mailbox has been deleted, not the children. The mailbox becomes \NOSELECT, and the hierarchy remains unchanged, as per the description of the DELETE command in IMAP4rev1 [RFC3501]. The parameters include the deleted mailbox identifier and MAY also indicate which access protocol triggered the event. MailboxRename A mailbox has been renamed. Note that, per the description of the RENAME command in IMAP4rev1 [RFC3501], special semantics regarding the mailbox hierarchy apply when INBOX is renamed (child mailboxes are usually included in the rename, but are excluded when INBOX is renamed). When a mailbox other than INBOX is renamed and its child mailboxes are also renamed as a result, separate MailboxRename events are not generated for the child mailboxes, as their renaming is implied. If the rename caused the creation of new mailboxes earlier in the hierarchy, separate MailboxCreate events are not generated for those, as their creation is implied. When INBOX is renamed, a new INBOX is created. A MailboxCreate event is not generated for the new INBOX, since it is implied. The parameters include the old mailbox identifier, the new mailbox identifier, and MAY also indicate which access protocol triggered the event. MailboxSubscribe A mailbox has been added to the server-stored subscription list, such as the one managed by the IMAP SUBSCRIBE and UNSUBSCRIBE commands. The parameters include the user whose subscription list has been affected, the mailbox identifier, and MAY also indicate which access protocol triggered the event. MailboxUnSubscribe A mailbox has been removed from the subscription list. The parameters include the user whose subscription list has been affected, the mailbox identifier, and MAY also indicate which access protocol triggered the event. Gellens & Newman Standards Track [Page 9] RFC 5423 Internet Message Store Events March 2009 5. Event Parameters This section lists parameters included with these events. admin Included with all events generated by message access protocols. The authentication identity associated with this event, as distinct from the authorization identity (see "user"). This is not included when it is the same as the value of the user parameter. bodyStructure May be included with MessageAppend and MessageNew. The IMAP BODYSTRUCTURE of the message. clientIP Included with all events generated by message access protocols. The IPv4 or IPv6 address of the message store access client that performed the action that triggered the notification. clientPort Included with all events generated by message access protocols. The port number of the message store access client that performed an action that triggered the notification (the port from which the connection occurred). diskQuota Included with QuotaExceed, QuotaWithin, and QuotaChange notifications relating to a user or mailbox disk quota. May be included with other notifications. Disk quota limit in kilobytes (1024 octets). diskUsed Included with QuotaExceed and QuotaWithin notifications relating to a user or mailbox disk quota. May be included with other notifications. Disk space used in kilobytes (1024 octets). Only disk space that counts against the quota is included. Gellens & Newman Standards Track [Page 10] RFC 5423 Internet Message Store Events March 2009 envelope May be included with the MessageNew notification. The message transfer envelope associated with final delivery of the message for the MessageNew notification. This includes the MAIL FROM and relevant RCPT TO line(s) used for final delivery with CRLF delimiters and any ESMTP parameters. flagNames Included with FlagsSet and FlagsClear events. May be included with MessageAppend and MessageNew to indicate flags that were set initially by the APPEND command or delivery agent, respectively. A list (likely to be space-separated) of IMAP flag or keyword names that were set or cleared. Flag names begin with a backslash while keyword names do not. The \Recent flag is explicitly not permitted in the list. mailboxID Included in events that affect mailboxes. A URI describing the mailbox. In the case of MailboxRename, this refers to the new name. maxMessages Included with QuotaExceed and QuotaWithin notifications relating to a user or mailbox message count quota. May be included with other notifications. Quota limit on the number of messages in the mailbox, for events referring to a mailbox. messageContent May be included with MessageAppend and MessageNew. The entire message itself. Size-based suppression of this SHOULD be available. messageSize May be included with MessageAppend and MessageNew. Size of the RFC 5322 message itself in octets. This value matches the length of the IMAP literal returned in response to an IMAP FETCH of BODY[] for the referenced message. Gellens & Newman Standards Track [Page 11] RFC 5423 Internet Message Store Events March 2009 messages Included with QuotaExceed and QuotaWithin notifications relating to a user or mailbox message count quota. May be included with other notifications. Number of messages in the mailbox. This is typically included with message addition and deletion events. modseq May be included with any notification referring to one message. This is the 64-bit integer MODSEQ as defined in [RFC4551]. No assumptions about MODSEQ can be made if this is omitted. oldMailboxID A URI describing the old name of a renamed or moved mailbox. pid May be included with any notification. The process ID of the process that generated the notification. process May be included with any notification. The name of the process that generated the notification. serverDomain Included in Login and optionally in Logout or other events. The domain name or IP address (v4 or v6) used to access the server or mailbox. serverPort Included in Login and optionally in Logout or other events. The port number used to access the server. This is often a well-known port. serverFQDN May be included with any notification. The fully qualified domain name of the server that generated the event. Note that this may be different from the server name used to access the mailbox included in the mailbox identifier. Gellens & Newman Standards Track [Page 12] RFC 5423 Internet Message Store Events March 2009 service May be included with any notification. The name of the service that triggered the event. Suggested values include "imap", "pop", "http", and "admincli" (for an administrative client). tags May be included with any notification. A list of UTF-8 tags (likely to be comma-separated). One or more tags can be set at the time a notification criteria or notification subscription is created. Subscribers can use tags for additional client-side filtering or dispatch of events. timestamp May be included with any notification. The time at which the event occurred that triggered the notification (the underlying protocol carrying the notification may contain a timestamp for when the notification was generated). This MAY be an approximate time. Timestamps are expressed in local time and contain the offset from UTC (this information is used in several places in Internet mail) and are normally in [RFC3339] format. uidnext May be included with any notification referring to a mailbox. The UID that is projected to be assigned next in the mailbox. This is typically included with message addition and deletion events. This is equivalent to the UIDNEXT status item in the IMAP STATUS command. uidset Included with MessageExpires, MessageExpunges, MessageRead, MessageTrash, FlagsSet, and FlagsClear. This includes the set of IMAP UIDs referenced. uri Included with all notifications. A reference to the IMAP server, a mailbox, or a message. Typically an IMAP URL. This can include the name of the server used to access the mailbox/message, the mailbox name, the UIDVALIDITY of the mailbox, and the UID of a specific message. Gellens & Newman Standards Track [Page 13] RFC 5423 Internet Message Store Events March 2009 user Included with all events generated by message access protocols. This is the authorization identifier used when the client connected to the access protocol that triggered the event. Some protocols (for example, many SASL mechanisms) distinguish between authorization and authentication identifiers. For events associated with a mailbox, this may be different from the owner of the mailbox specified in the IMAP URL. 6. IANA Considerations The IANA has created a new registry for "Internet Message Store Events" that contains two sub-registries: event names and event parameters. For both event names and event parameters, entries that do not start with "vnd." are added by the IETF and are intended for interoperable use. Entries that start with "vnd." are intended for private use by one or more parties and are allocated to avoid collisions. The initial values are contained in this document. Using IANA Considerations [RFC5226] terminology, entries that do not start with "vnd." are allocated by IETF Consensus, while those starting with "vnd." are allocated First Come First Served. 7. Security Considerations Notifications can produce a large amount of traffic and expose sensitive information. When notification mechanisms are used to maintain state between different entities, the ability to corrupt or manipulate notification messages could enable an attacker to modulate the state of these entities. For example, if an attacker were able to modify notifications sent from a message store to an auditing server, he could modify the "user" and "messageContent" parameters in MessageNew notifications to create false audit log entries. A competent transfer protocol for notifications must consider authentication, authorization, privacy, and message integrity, as well as denial-of-service issues. While the IETF has adequate tools and experience to address these issues for mechanisms that involve only one TCP connection, notification or publish/subscribe protocols that are more sophisticated than a single end-to-end TCP connection will need to pay extra attention to these issues and carefully balance requirements to successfully deploy a system with security and privacy considerations. Gellens & Newman Standards Track [Page 14] RFC 5423 Internet Message Store Events March 2009 8. Acknowledgments Alexey Melnikov, Arnt Gulbrandsen, and Zoltan Ordogh have reviewed and offered improvements to this document. Richard Barnes did a nice review during Last Call. 9. References 9.1. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC5092] Melnikov, A. and C. Newman, "IMAP URL Scheme", RFC 5092, November 2007. [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an IANA Considerations Section in RFCs", BCP 26, RFC 5226, May 2008. 9.2. Informative References [RFC1939] Myers, J. and M. Rose, "Post Office Protocol - Version 3", STD 53, RFC 1939, May 1996. [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. [RFC2447] Dawson, F., Mansour, S., and S. Silverberg, "iCalendar Message-Based Interoperability Protocol (iMIP)", RFC 2447, November 1998. [RFC2616] Fielding, R., Gettys, J., Mogul, J., Frystyk, H., Masinter, L., Leach, P., and T. Berners-Lee, "Hypertext Transfer Protocol -- HTTP/1.1", RFC 2616, June 1999. [RFC3339] Klyne, G., Ed. and C. Newman, "Date and Time on the Internet: Timestamps", RFC 3339, July 2002. [RFC3458] Burger, E., Candell, E., Eliot, C., and G. Klyne, "Message Context for Internet Mail", RFC 3458, January 2003. [RFC4146] Gellens, R., "Simple New Mail Notification", RFC 4146, August 2005. Gellens & Newman Standards Track [Page 15] RFC 5423 Internet Message Store Events March 2009 [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and Security Layer (SASL)", RFC 4422, June 2006. [RFC4467] Crispin, M., "Internet Message Access Protocol (IMAP) - URLAUTH Extension", RFC 4467, May 2006. [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, October 2008. Gellens & Newman Standards Track [Page 16] RFC 5423 Internet Message Store Events March 2009 Appendix A. Future Extensions This document specifies core functionality based on events that are believed to be well understood, have known use cases, and are implemented by at least one deployed real-world Internet message store. (A few events are exceptions to the last test only: FlagsSet, FlagsClear, MailboxCreate, MailboxDelete, MailboxRename, MailboxSubscribe, and MailboxUnSubscribe.) Some events have been suggested but are postponed to future extensions because they do not meet this criteria. These events include messages that have been moved to archive storage and may require extra time to access, quota by message context, authentication failure, user mail account disabled, annotations, and mailbox ACL or metadata change. The descriptions of several events note additional parameters that are likely candidates for future inclusion. See Section 6 for how the list of events and parameters can be extended. In order to narrow the scope of this document to something that can be completed, only events generated from the message store (by a message access module, administrative module, or message delivery agent) are considered. A complete mail system is normally linked with an identity system that would also publish events of interest to a message store event subscriber. Events of interest include account created/deleted/disabled and password changed/expired. Authors' Addresses Randall Gellens QUALCOMM Incorporated 5775 Morehouse Drive San Diego, CA 92651 USA Phone: EMail: rg+ietf@qualcomm.com Chris Newman Sun Microsystems 800 Royal Oaks Monrovia, CA 91016-6347 USA Phone: EMail: chris.newman@sun.com Gellens & Newman Standards Track [Page 17] ================================================ FILE: docs/rfcs/rfc5464.IMAP_METADATA_extension.txt ================================================ Network Working Group C. Daboo Request for Comments: 5464 Apple, Inc. Category: Standards Track February 2009 The IMAP METADATA Extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Abstract The METADATA extension to the Internet Message Access Protocol permits clients and servers to maintain "annotations" or "metadata" on IMAP servers. It is possible to have annotations on a per-mailbox basis or on the server as a whole. For example, this would allow comments about the purpose of a particular mailbox to be "attached" to that mailbox, or a "message of the day" containing server status information to be made available to anyone logging in to the server. Daboo Standards Track [Page 1] RFC 5464 The IMAP METADATA Extension February 2009 Table of Contents 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 3 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 3. Data Model . . . . . . . . . . . . . . . . . . . . . . . . . . 4 3.1. Overview . . . . . . . . . . . . . . . . . . . . . . . . . 4 3.2. Namespace of Entries . . . . . . . . . . . . . . . . . . . 4 3.2.1. Entry Names . . . . . . . . . . . . . . . . . . . . . 5 3.3. Private versus Shared and Access Control . . . . . . . . . 6 4. IMAP Protocol Changes . . . . . . . . . . . . . . . . . . . . 7 4.1. General Considerations . . . . . . . . . . . . . . . . . . 7 4.2. GETMETADATA Command . . . . . . . . . . . . . . . . . . . 8 4.2.1. MAXSIZE GETMETADATA Command Option . . . . . . . . . . 9 4.2.2. DEPTH GETMETADATA Command Option . . . . . . . . . . . 10 4.3. SETMETADATA Command . . . . . . . . . . . . . . . . . . . 10 4.4. METADATA Response . . . . . . . . . . . . . . . . . . . . 12 4.4.1. METADATA Response with Values . . . . . . . . . . . . 13 4.4.2. Unsolicited METADATA Response without Values . . . . . 13 5. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 14 6. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 16 6.1. Entry and Attribute Registration Template . . . . . . . . 16 6.2. Server Entry Registrations . . . . . . . . . . . . . . . . 16 6.2.1. /shared/comment . . . . . . . . . . . . . . . . . . . 17 6.2.2. /shared/admin . . . . . . . . . . . . . . . . . . . . 17 6.3. Mailbox Entry Registrations . . . . . . . . . . . . . . . 17 6.3.1. /shared/comment . . . . . . . . . . . . . . . . . . . 18 6.3.2. /private/comment . . . . . . . . . . . . . . . . . . . 18 7. Security Considerations . . . . . . . . . . . . . . . . . . . 18 8. Normative References . . . . . . . . . . . . . . . . . . . . . 19 Appendix A. Acknowledgments . . . . . . . . . . . . . . . . . . . 19 Daboo Standards Track [Page 2] RFC 5464 The IMAP METADATA Extension February 2009 1. Introduction and Overview The goal of the METADATA extension is to provide a means for clients to set and retrieve "annotations" or "metadata" on an IMAP server. The annotations can be associated with specific mailboxes or the server as a whole. The server can choose to support only server annotations or both server and mailbox annotations. A server that supports both server and mailbox annotations indicates the presence of this extension by returning "METADATA" as one of the supported capabilities in the CAPABILITY command response. A server that supports only server annotations indicates the presence of this extension by returning "METADATA-SERVER" as one of the supported capabilities in the CAPABILITY command response. A server that supports unsolicited annotation change responses MUST support the "ENABLE" [RFC5161] extension to allow clients to turn that feature on. The METADATA extension adds two new commands and one new untagged response to the IMAP base protocol. This extension makes the following changes to the IMAP protocol: o adds a new SETMETADATA command o adds a new GETMETADATA command o adds a new METADATA untagged response o adds a new METADATA response code The rest of this document describes the data model and protocol changes more rigorously. 2. Conventions Used in This Document In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. Whitespace and line breaks have been added to the examples in this document to promote readability. Daboo Standards Track [Page 3] RFC 5464 The IMAP METADATA Extension February 2009 3. Data Model 3.1. Overview Mailboxes or the server as a whole may have zero or more annotations associated with them. An annotation contains a uniquely named entry, which has a value. Annotations can be added to mailboxes when a mailbox name is provided as the first argument to the SETMETADATA command, or to the server as a whole when the empty string is provided as the first argument to the command. For example, a general comment being added to a mailbox may have an entry name of "/comment" and a value of "Really useful mailbox". The protocol changes to IMAP described below allow a client to access or change the values of any annotation entry, assuming it has sufficient access rights to do so. 3.2. Namespace of Entries Each annotation is an entry that has a hierarchical name, with each component of the name separated by a slash ("/"). An entry name MUST NOT contain two consecutive "/" characters and MUST NOT end with a "/" character. The value of an entry is NIL (has no value), or a string or binary data of zero or more octets. A string MAY contain multiple lines of text. Clients MUST use the CRLF (0x0D 0x0A) character octet sequence to represent line ends in a multi-line string value. Entry names MUST NOT contain asterisk ("*") or percent ("%") characters and MUST NOT contain non-ASCII characters or characters with octet values in the range 0x00 to 0x19. Invalid entry names result in a BAD response in any IMAP command in which they are used. Entry names are case-insensitive. Use of control or punctuation characters in entry names is strongly discouraged. This specification defines an initial set of entry names available for use with mailbox and server annotations. In addition, an extension mechanism is described to allow additional names to be added for extensibility. The first component in entry names defines the scope of the annotation. Currently, only the prefixes "/private" or "/shared" are defined. These prefixes are used to indicate whether an annotation Daboo Standards Track [Page 4] RFC 5464 The IMAP METADATA Extension February 2009 is stored on a per-user basis ("/private") and not visible to other users, or whether an annotation is shared between authorized users ("/shared") with a single value that can be read and changed by authorized users with appropriate access. See Section 3.3 for details. Entry names can have any number of components starting at 2, unless they fall under the vendor namespaces (i.e., have a /shared/vendor/ or /private/vendor/ prefix as described below), in which case they have at least 4 components. 3.2.1. Entry Names Entry names MUST be specified in a Standards Track or IESG-approved Experimental RFC, or fall under the vendor namespace. See Section 6.1 for the registration template. 3.2.1.1. Server Entries These entries are set or retrieved when the mailbox name argument to the new SETMETADATA or GETMETADATA command is the empty string. /shared/comment Defines a comment or note that is associated with the server and that is shared with authorized users of the server. /shared/admin Indicates a method for contacting the server administrator. The value MUST be a URI (e.g., a mailto: or tel: URL). This entry is always read-only -- clients cannot change it. It is visible to authorized users of the system. /shared/vendor/ Defines the top level of shared entries associated with the server, as created by a particular product of some vendor. This entry can be used by vendors to provide server- or client-specific annotations. The vendor-token MUST be registered with IANA, using the Application Configuration Access Protocol (ACAP) [RFC2244] vendor subtree registry. /private/vendor/ Defines the top level of private entries associated with the server, as created by a particular product of some vendor. This entry can be used by vendors to provide server- or client-specific Daboo Standards Track [Page 5] RFC 5464 The IMAP METADATA Extension February 2009 annotations. The vendor-token MUST be registered with IANA, using the ACAP [RFC2244] vendor subtree registry. 3.2.1.2. Mailbox Entries These entries are set or retrieved when the mailbox name argument to the new SETMETADATA or GETMETADATA command is not the empty string. /shared/comment Defines a shared comment or note associated with a mailbox. /private/comment Defines a private (per-user) comment or note associated with a mailbox. /shared/vendor/ Defines the top level of shared entries associated with a specific mailbox, as created by a particular product of some vendor. This entry can be used by vendors to provide client-specific annotations. The vendor-token MUST be registered with IANA, using the ACAP [RFC2244] vendor subtree registry. /private/vendor/ Defines the top level of private entries associated with a specific mailbox, as created by a particular product of some vendor. This entry can be used by vendors to provide client- specific annotations. The vendor-token MUST be registered with IANA, using the ACAP [RFC2244] vendor subtree registry. 3.3. Private versus Shared and Access Control In the absence of the ACL (Access Control List) extension [RFC4314], users can only set and retrieve private or shared mailbox annotations on a mailbox that exists and is returned to them via a LIST or LSUB command, and on which they have either read or write access to the actual message content of the mailbox (as determined by the READ-ONLY and READ-WRITE response codes as described in Section 5.2 of [RFC4314]). When the ACL extension [RFC4314] is present, users can only set and retrieve private or shared mailbox annotations on a mailbox on which they have the "l" right and any one of the "r", "s", "w", "i", or "p" rights. Daboo Standards Track [Page 6] RFC 5464 The IMAP METADATA Extension February 2009 If a client attempts to set or retrieve annotations on mailboxes that do not satisfy the conditions above, the server MUST respond with a NO response. Users can always retrieve private or shared server annotations if they exist. Servers MAY restrict the creation of private or shared server annotations as appropriate. When restricted, the server MUST return a NO response when the SETMETADATA command is used to try to create a server annotation. If the METADATA extension is present, support for shared annotations is REQUIRED, whilst support for private annotations is OPTIONAL. This recognizes the fact that support for private annotations may introduce significantly more complexity to a server in terms of tracking ownership of the annotations, how quota is determined for users based on their own annotations, etc. 4. IMAP Protocol Changes 4.1. General Considerations The new SETMETADATA command and the METADATA response each have a mailbox name argument. An empty string is used for the mailbox name to signify server annotations. A non-empty string is used to signify mailbox annotations attached to the corresponding mailbox. Servers SHOULD ensure that mailbox annotations are automatically moved when the mailbox they refer to is renamed, i.e., the annotations follow the mailbox. This applies to a rename of the INBOX, with the additional behavior that the annotations are copied from the original INBOX to the renamed mailbox, i.e., mailbox annotations are preserved on the INBOX when it is renamed. Servers SHOULD delete annotations for a mailbox when the mailbox is deleted, so that a mailbox created with the same name as a previously existing mailbox does not inherit the old mailbox annotations. Servers SHOULD allow annotations on all 'types' of mailboxes, including ones reporting \Noselect for their LIST response. Servers can implicitly remove \Noselect mailboxes when all child mailboxes are removed, and, at that time any annotations associated with the \Noselect mailbox SHOULD be removed. The server is allowed to impose limitations on the size of any one annotation or the total number of annotations for a single mailbox or for the server as a whole. However, the server MUST accept an annotation data size of at least 1024 bytes, and an annotation count per server or mailbox of at least 10. Daboo Standards Track [Page 7] RFC 5464 The IMAP METADATA Extension February 2009 Some annotations may be "read-only" -- i.e., they are set by the server and cannot be changed by the client. Also, such annotations may be "computed" -- i.e., the value changes based on underlying properties of the mailbox or server. For example, an annotation reporting the total size of all messages in the mailbox would change as messages are added or removed. Or, an annotation containing an IMAP URL for the mailbox would change if the mailbox was renamed. Servers MAY support sending unsolicited responses for use when annotations are changed by some "third-party" (see Section 4.4). In order to do so, servers MUST support the ENABLE command [RFC5161] and MUST only send unsolicited responses if the client used the ENABLE command [RFC5161] extension with the capability string "METADATA" or "METADATA-SERVER" earlier in the session, depending on which of those capabilities is supported by the server. 4.2. GETMETADATA Command This extension adds the GETMETADATA command. This allows clients to retrieve server or mailbox annotations. This command is only available in authenticated or selected state [RFC3501]. Arguments: mailbox-name options entry-specifier Responses: required METADATA response Result: OK - command completed NO - command failure: can't access annotations on the server BAD - command unknown or arguments invalid When the mailbox name is the empty string, this command retrieves server annotations. When the mailbox name is not empty, this command retrieves annotations on the specified mailbox. Options MAY be included with this command and are defined below. Example: C: a GETMETADATA "" /shared/comment S: * METADATA "" (/shared/comment "Shared comment") S: a OK GETMETADATA complete Daboo Standards Track [Page 8] RFC 5464 The IMAP METADATA Extension February 2009 In the above example, the contents of the value of the "/shared/ comment" server entry is requested by the client and returned by the server. Example: C: a GETMETADATA "INBOX" /private/comment S: * METADATA "INBOX" (/private/comment "My own comment") S: a OK GETMETADATA complete In the above example, the contents of the value of the "/private/ comment" mailbox entry for the mailbox "INBOX" is requested by the client and returned by the server. Entry specifiers can be lists of atomic specifiers, so that multiple annotations may be returned in a single GETMETADATA command. Example: C: a GETMETADATA "INBOX" (/shared/comment /private/comment) S: * METADATA "INBOX" (/shared/comment "Shared comment" /private/comment "My own comment") S: a OK GETMETADATA complete In the above example, the values of the two server entries "/shared/comment" and "/private/comment" on the mailbox "INBOX" are requested by the client and returned by the server. 4.2.1. MAXSIZE GETMETADATA Command Option When the MAXSIZE option is specified with the GETMETADATA command, it restricts which entry values are returned by the server. Only entry values that are less than or equal in octet size to the specified MAXSIZE limit are returned. If there are any entries with values larger than the MAXSIZE limit, the server MUST include the METADATA LONGENTRIES response code in the tagged OK response for the GETMETADATA command. The METADATA LONGENTRIES response code returns the size of the biggest entry value requested by the client that exceeded the MAXSIZE limit. Example: C: a GETMETADATA "INBOX" (MAXSIZE 1024) (/shared/comment /private/comment) S: * METADATA "INBOX" (/private/comment "My own comment") S: a OK [METADATA LONGENTRIES 2199] GETMETADATA complete Daboo Standards Track [Page 9] RFC 5464 The IMAP METADATA Extension February 2009 In the above example, the values of the two server entries "/shared/comment" and "/private/comment" on the mailbox "INBOX" are requested by the client, which wants to restrict the size of returned values to 1024 octets. In this case, the "/shared/ comment" entry value is 2199 octets and is not returned. 4.2.2. DEPTH GETMETADATA Command Option When the DEPTH option is specified with the GETMETADATA command, it extends the list of entry values returned by the server. For each entry name specified in the GETMETADATA command, the server returns the value of the specified entry name (if it exists), plus all entries below the entry name up to the specified DEPTH. Three values are allowed for DEPTH: "0" - no entries below the specified entry are returned "1" - only entries immediately below the specified entry are returned "infinity" - all entries below the specified entry are returned Thus, "depth 1" for an entry "/a" will match "/a" as well as its children entries (e.g., "/a/b"), but will not match grandchildren entries (e.g., "/a/b/c"). If the DEPTH option is not specified, this is the same as specifying "DEPTH 0". Example: C: a GETMETADATA "INBOX" (DEPTH 1) (/private/filters/values) S: * METADATA "INBOX" (/private/filters/values/small "SMALLER 5000" /private/filters/values/boss "FROM \"boss@example.com\"") S: a OK GETMETADATA complete In the above example, 2 entries below the /private/filters/values entry exist on the mailbox "INBOX": "/private/filters/values/ small" and "/private/filters/values/boss". 4.3. SETMETADATA Command This extension adds the SETMETADATA command. This allows clients to set annotations. This command is only available in authenticated or selected state [RFC3501]. Daboo Standards Track [Page 10] RFC 5464 The IMAP METADATA Extension February 2009 Arguments: mailbox-name entry value list of entry, values Responses: no specific responses for this command Result: OK - command completed NO - command failure: can't set annotations, or annotation too big or too many BAD - command unknown or arguments invalid This command sets the specified list of entries by adding or replacing the specified values provided, on the specified existing mailboxes or on the server (if the mailbox argument is the empty string). Clients can use NIL for the value of entries it wants to remove. The server SHOULD NOT return a METADATA response containing the updated annotation data. Clients MUST NOT assume that a METADATA response will be sent, and MUST assume that if the command succeeds, then the annotation has been changed. If the server is unable to set an annotation because the size of its value is too large, the server MUST return a tagged NO response with a "[METADATA MAXSIZE NNN]" response code when NNN is the maximum octet count that it is willing to accept. If the server is unable to set a new annotation because the maximum number of allowed annotations has already been reached, the server MUST return a tagged NO response with a "[METADATA TOOMANY]" response code. If the server is unable to set a new annotation because it does not support private annotations on one of the specified mailboxes, the server MUST return a tagged NO response with a "[METADATA NOPRIVATE]" response code. When any one annotation fails to be set, resulting in a tagged NO response from the server, then the server MUST NOT change the values for other annotations specified in the SETMETADATA command. Example: C: a SETMETADATA INBOX (/private/comment {33} S: + ready for data My new comment across two lines. ) S: a OK SETMETADATA complete Daboo Standards Track [Page 11] RFC 5464 The IMAP METADATA Extension February 2009 In the above example, the entry "/private/comment" for the mailbox "INBOX" is created (if not already present) and the value set to a multi-line string. Example: C: a SETMETADATA INBOX (/private/comment NIL) S: a OK SETMETADATA complete In the above example, the entry "/private/comment" is removed from the mailbox "INBOX". Multiple entries can be set in a single SETMETADATA command by listing entry-value pairs in the list. Example: C: a SETMETADATA INBOX (/private/comment "My new comment" /shared/comment "This one is for you!") S: a OK SETMETADATA complete In the above example, the entries "/private/comment" and "/shared/ comment" for the mailbox "INBOX" are created (if not already present) and the values set as specified. Example: C: a SETMETADATA INBOX (/private/comment "My new comment") S: a NO [METADATA TOOMANY] SETMETADATA failed In the above example, the server is unable to set the requested (new) annotation as it has reached the limit on the number of annotations it can support on the specified mailbox. 4.4. METADATA Response The METADATA response displays results of a GETMETADATA command, or can be returned as an unsolicited response at any time by the server in response to a change in a server or mailbox annotation. When unsolicited responses are activated by the ENABLE [RFC5161] command for this extension, servers MUST send unsolicited METADATA responses if server or mailbox annotations are changed by a third- party, allowing servers to keep clients updated with changes. Unsolicited METADATA responses MUST only contain entry names, not the values. If the client wants to update any cached values, it must explicitly retrieve those using a GETMETADATA command. Daboo Standards Track [Page 12] RFC 5464 The IMAP METADATA Extension February 2009 The METADATA response can contain multiple entries in a single response, but the server is free to return multiple responses for each entry or group of entries, if it desires. This response is only available in authenticated or selected state [RFC3501]. 4.4.1. METADATA Response with Values The response consists of a list of entry-value pairs. Example: C: a GETMETADATA "" /shared/comment S: * METADATA "" (/shared/comment "My comment") S: a OK GETMETADATA complete In the above example, a single entry with its value is returned by the server. Example: C: a GETMETADATA "INBOX" /private/comment /shared/comment S: * METADATA "INBOX" (/private/comment "My comment" /shared/comment "Its sunny outside!") S: a OK GETMETADATA complete In the above example, two entries and their values are returned by the server. Example: C: a GETMETADATA "INBOX" /private/comment /shared/comment S: * METADATA "INBOX" (/private/comment "My comment") S: * METADATA "INBOX" (/shared/comment "Its sunny outside!") S: a OK GETMETADATA complete In the above example, the server returns two separate responses for each of the two entries requested. 4.4.2. Unsolicited METADATA Response without Values The response consists of a list of entries, each of which have changed on the server or mailbox. Example: Daboo Standards Track [Page 13] RFC 5464 The IMAP METADATA Extension February 2009 C: a NOOP S: * METADATA "" /shared/comment S: a OK NOOP complete In the above example, the server indicates that the "/shared/ comment" server entry has been changed. Example: C: a NOOP S: * METADATA "INBOX" /shared/comment /private/comment S: a OK NOOP complete In the above example, the server indicates a change to two mailbox entries. 5. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234]. Non-terminals referenced but not defined below are as defined by [RFC3501], with the new definitions in [RFC4466] superseding those in [RFC3501]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. capability =/ "METADATA" / "METADATA-SERVER" ; defines the capabilities for this extension. command-auth =/ setmetadata / getmetadata ; adds to original IMAP command entries = entry / "(" entry *(SP entry) ")" ; entry specifiers entry = astring ; slash-separated path to entry ; MUST NOT contain "*" or "%" entry-value = entry SP value entry-values = "(" entry-value *(SP entry-value) ")" Daboo Standards Track [Page 14] RFC 5464 The IMAP METADATA Extension February 2009 entry-list = entry *(SP entry) ; list of entries used in unsolicited ; METADATA response getmetadata = "GETMETADATA" [SP getmetadata-options] SP mailbox SP entries ; empty string for mailbox implies ; server annotation. getmetadata-options = "(" getmetadata-option *(SP getmetadata-option) ")" getmetadata-option = tagged-ext-label [SP tagged-ext-val] ; tagged-ext-label and tagged-ext-val ; are defined in [RFC4466]. maxsize-opt = "MAXSIZE" SP number ; Used as a getmetadata-option metadata-resp = "METADATA" SP mailbox SP (entry-values / entry-list) ; empty string for mailbox implies ; server annotation. response-payload =/ metadata-resp ; adds to original IMAP data responses resp-text-code =/ "METADATA" SP "LONGENTRIES" SP number ; new response codes for GETMETADATA resp-text-code =/ "METADATA" SP ("MAXSIZE" SP number / "TOOMANY" / "NOPRIVATE") ; new response codes for SETMETADATA ; failures scope-opt = "DEPTH" SP ("0" / "1" / "infinity") ; Used as a getmetadata-option setmetadata = "SETMETADATA" SP mailbox SP entry-values ; empty string for mailbox implies ; server annotation. value = nstring / literal8 Daboo Standards Track [Page 15] RFC 5464 The IMAP METADATA Extension February 2009 6. IANA Considerations All entries MUST have either "/shared" or "/private" as a prefix. Entry names MUST be specified in a Standards Track or IESG-approved Experimental RFC, or fall under the vendor namespace (i.e., use /shared/vendor/ or /private/vendor/ as the prefix). Each entry registration MUST include a content-type that is used to indicate the nature of the annotation value. Where applicable, a charset parameter MUST be included with the content-type. 6.1. Entry and Attribute Registration Template To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: [Either "Mailbox" or "Server"] Name: [the name of the entry] Description: [a description of what the entry is for] Content-type: [MIME Content-Type and charset for the entry value] RFC Number: [for entries published as RFCs] Contact: [email and/or physical address to contact for additional information] 6.2. Server Entry Registrations The following templates specify the IANA registrations of annotation entries specified in this document. Daboo Standards Track [Page 16] RFC 5464 The IMAP METADATA Extension February 2009 6.2.1. /shared/comment To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: Server Name: /shared/comment Description: Defines a comment or note that is associated with the server and that is shared with authorized users of the server. Content-type: text/plain; charset=utf-8 RFC Number: RFC 5464 Contact: IMAP Extensions mailto:ietf-imapext@imc.org 6.2.2. /shared/admin To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: Server Name: /shared/admin Description: Indicates a method for contacting the server administrator. The value MUST be a URI (e.g., a mailto: or tel: URL). This entry is always read-only -- clients cannot change it. It is visible to authorized users of the system. Content-type: text/plain; charset=utf-8 RFC Number: RFC 5464 Contact: IMAP Extensions mailto:ietf-imapext@imc.org 6.3. Mailbox Entry Registrations The following templates specify the IANA registrations of annotation entries specified in this document. Daboo Standards Track [Page 17] RFC 5464 The IMAP METADATA Extension February 2009 6.3.1. /shared/comment To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: Mailbox Name: /shared/comment Description: Defines a shared comment or note associated with a mailbox. Content-type: text/plain; charset=utf-8 RFC Number: RFC 5464 Contact: IMAP Extensions mailto:ietf-imapext@imc.org 6.3.2. /private/comment To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: Mailbox Name: /private/comment Description: Defines a private comment or note associated with a mailbox. Content-type: text/plain; charset=utf-8 RFC Number: RFC 5464 Contact: IMAP Extensions mailto:ietf-imapext@imc.org 7. Security Considerations The security considerations in Section 11 of [RFC3501] apply here with respect to protecting annotations from snooping. Servers MAY choose to only support the METADATA and/or METADATA-SERVER extensions after a privacy layer has been negotiated by the client. Annotations can contain arbitrary data of varying size. As such, servers MUST ensure that size limits are enforced to prevent a user from using up all available space on a server and preventing use by others. Clients MUST treat annotation data values as an "untrusted" source of data as it is possible for it to contain malicious content. Daboo Standards Track [Page 18] RFC 5464 The IMAP METADATA Extension February 2009 Annotations whose values are intended to remain private MUST be stored only in entries that have the "/private" prefix on the entry name. Excluding the above issues, the METADATA extension does not raise any security considerations that are not present in the base IMAP protocol, and these issues are discussed in [RFC3501]. 8. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC2244] Newman, C. and J. Myers, "ACAP -- Application Configuration Access Protocol", RFC 2244, November 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE Extension", RFC 5161, March 2008. [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. Appendix A. Acknowledgments The ideas expressed in this document are based on the message annotation document that was co-authored by Randall Gellens. The author would like to thank the following individuals for contributing their ideas and support for writing this specification: Dave Cridland, Arnt Gulbrandsen, Dan Karp, Alexey Melnikov, Ken Murchison, Chris Newman, and Michael Wener. Daboo Standards Track [Page 19] RFC 5464 The IMAP METADATA Extension February 2009 Author's Address Cyrus Daboo Apple Inc. 1 Infinite Loop Cupertino, CA 95014 USA EMail: cyrus@daboo.name URI: http://www.apple.com/ Daboo Standards Track [Page 20] RFC 5464 The IMAP METADATA Extension February 2009 Full Copyright Statement Copyright (C) The IETF Trust (2009). This document is subject to the rights, licenses and restrictions contained in BCP 78, and except as set forth therein, the authors retain all their rights. This document and the information contained herein are provided on an "AS IS" basis and THE CONTRIBUTOR, THE ORGANIZATION HE/SHE REPRESENTS OR IS SPONSORED BY (IF ANY), THE INTERNET SOCIETY, THE IETF TRUST AND THE INTERNET ENGINEERING TASK FORCE DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Intellectual Property The IETF takes no position regarding the validity or scope of any Intellectual Property Rights or other rights that might be claimed to pertain to the implementation or use of the technology described in this document or the extent to which any license under such rights might or might not be available; nor does it represent that it has made any independent effort to identify any such rights. Information on the procedures with respect to rights in RFC documents can be found in BCP 78 and BCP 79. Copies of IPR disclosures made to the IETF Secretariat and any assurances of licenses to be made available, or the result of an attempt made to obtain a general license or permission for the use of such proprietary rights by implementers or users of this specification can be obtained from the IETF on-line IPR repository at http://www.ietf.org/ipr. The IETF invites any interested party to bring to its attention any copyrights, patents or patent applications, or other proprietary rights that may cover technology that may be required to implement this standard. Please address the information to the IETF at ietf-ipr@ietf.org. Daboo Standards Track [Page 21] ================================================ FILE: docs/rfcs/rfc5465.IMAP_NOTIFY_extension.txt ================================================ Network Working Group A. Gulbrandsen Request for Comments: 5465 Oryx Mail Systems GmbH Updates: 5267 C. King Category: Standards Track A. Melnikov Isode Ltd. February 2009 The IMAP NOTIFY Extension Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (c) 2009 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/ license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Abstract This document defines an IMAP extension that allows a client to request specific kinds of unsolicited notifications for specified mailboxes, such as messages being added to or deleted from such mailboxes. Gulbrandsen, et al. Standards Track [Page 1] RFC 5465 IMAP NOTIFY Extension February 2009 Table of Contents 1. Overview and Rationale ..........................................3 2. Conventions Used in This Document ...............................4 3. The NOTIFY Extension ............................................4 3.1. The NOTIFY Command .........................................4 4. Interaction with the IDLE Command ...............................8 5. Event Types .....................................................8 5.1. FlagChange and AnnotationChange ............................9 5.2. MessageNew .................................................9 5.3. MessageExpunge ............................................10 5.4. MailboxName ...............................................11 5.5. SubscriptionChange ........................................12 5.6. MailboxMetadataChange .....................................12 5.7. ServerMetadataChange ......................................13 5.8. Notification Overflow .....................................13 5.9. ACL (Access Control List) Changes .........................13 6. Mailbox Specification ..........................................14 6.1. Mailbox Specifiers Affecting the Currently Selected Mailbox ..........................................14 6.2. Personal ..................................................15 6.3. Inboxes ...................................................15 6.4. Subscribed ................................................15 6.5. Subtree ...................................................15 6.6. Mailboxes .................................................16 7. Extension to SEARCH and SORT Commands ..........................16 8. Formal Syntax ..................................................16 9. Security Considerations ........................................19 10. IANA Considerations ...........................................19 10.1. Initial LIST-EXTENDED Extended Data Item Registrations ...19 11. Acknowledgements ..............................................20 12. Normative References ..........................................20 13. Informative References ........................................21 Gulbrandsen, et al. Standards Track [Page 2] RFC 5465 IMAP NOTIFY Extension February 2009 1. Overview and Rationale The IDLE command (defined in [RFC2177]) provides a way for the client to go into a mode where the IMAP server pushes it notifications about IMAP mailstore events for the selected mailbox. However, the IDLE extension doesn't restrict or control which server events can be sent, or what information the server sends in response to each event. Also, IDLE only applies to the selected mailbox, thus requiring an additional TCP connection per mailbox. This document defines an IMAP extension that allows clients to express their preferences about unsolicited events generated by the server. The extension allows clients to only receive events that they are interested in, while servers know that they don't need to go to the effort of generating certain types of untagged responses. Without the NOTIFY command defined in this document, an IMAP server will only send information about mailstore changes to the client in the following cases: - as the result of a client command (e.g., FETCH responses to a FETCH or STORE command), - as unsolicited responses sent just before the end of a command (e.g., EXISTS or EXPUNGE) as the result of changes in other sessions, and - during an IDLE command. The NOTIFY command extends what information may be returned in those last two cases, and also permits and requires the server to send information about updates between commands. The NOTIFY command also allows for the client to extend what information is sent unsolicited about the selected mailbox and to request some update information to be sent regarding other mailboxes. The interaction between IDLE and NOTIFY commands is described in Section 4. For the new messages delivered to or appended to the selected mailbox, the NOTIFY command can be used to request that a set of attributes be sent to the client in an unsolicited FETCH response. This allows a client to be a passive recipient of events and new mail and to be able to maintain full synchronisation without having to issue any subsequent commands except to modify the state of the mailbox on the server. Gulbrandsen, et al. Standards Track [Page 3] RFC 5465 IMAP NOTIFY Extension February 2009 Some mobile clients, however, may want mail "pushed" only for mail that matches a SEARCH pattern. To meet that need, [RFC5267] is augmented by this document to extend the UPDATE return option to specify a list of fetch-atts to be returned when a new message is delivered or appended in another session. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. The acronym MSN stands for Message Sequence Numbers (see Section 2.3.1.2 of [RFC3501]). Example lines prefaced by "C:" are sent by the client and ones prefaced by "S:", by the server. "[...]" means elision. 3. The NOTIFY Extension IMAP servers that support this extension advertise the NOTIFY capability. This extension adds the NOTIFY command as defined in Section 5.1. A server implementing this extension is not required to implement LIST-EXTENDED [RFC5258], even though a NOTIFY-compliant server must be able to return extended LIST responses, defined in [RFC5258]. 3.1. The NOTIFY Command Arguments: "SET" Optional STATUS indicator Mailboxes to be watched Events about which to notify the client Or Arguments: "NONE" Responses: Possibly untagged STATUS responses (for SET) Result: OK - The server will notify the client as requested. NO - Unsupported NOTIFY event, NOTIFY too complex or expensive, etc. BAD - Command unknown, invalid, unsupported, or has unknown arguments. Gulbrandsen, et al. Standards Track [Page 4] RFC 5465 IMAP NOTIFY Extension February 2009 The NOTIFY command informs the server that the client listens for event notifications all the time (even when no command is in progress), and requests the server to notify it about the specified set of events. The NOTIFY command has two forms. NOTIFY NONE specifies that the client is not interested in any kind of event happening on the server. NOTIFY SET replaces the current list of interesting events with a new list of events. Until the NOTIFY command is used for the first time, the server only sends notifications while a command is being processed, and notifies the client about these events on the selected mailbox (see Section 5 for definitions): MessageNew, MessageExpunge, or FlagChange. It does not notify the client about any events on other mailboxes. The effect of a successful NOTIFY command lasts until the next NOTIFY command or until the IMAP connection is closed. A successful NOTIFY SET command MUST cause the server to immediately return any accumulated changes to the currently selected mailbox (if any), such as flag changes and new or expunged messages. Thus, a successful NOTIFY SET command implies an implicit NOOP command. The NOTIFY SET command can request notifications of message-related changes to the selected mailbox, whatever that may be at the time the message notifications are being generated. This is done by specifying either the SELECTED or the SELECTED-DELAYED mailbox selector (see Section 6.1) in the NOTIFY SET command. If the SELECTED/SELECTED-DELAYED mailbox selector is not specified in the NOTIFY SET command, this means that the client doesn't want to receive any s for the currently selected mailbox. This is the same as specifying SELECTED NONE. The client can also request notifications on other mailboxes by name or by a limited mailbox pattern match. Message-related notifications returned for the currently selected mailbox will be those specified by the SELECTED/SELECTED-DELAYED mailbox specifier, even if the selected mailbox also appears by name (or matches a pattern) in the command. Non-message-related notifications are controlled by mailbox specifiers other than SELECTED/SELECTED-DELAYED. If the NOTIFY command enables MessageNew, MessageExpunge, AnnotationChange, or FlagChange notifications for a mailbox other than the currently selected mailbox, and the client has specified the STATUS indicator parameter, then the server MUST send a STATUS response for that mailbox before NOTIFY's tagged OK. If MessageNew is enabled, the STATUS response MUST contain MESSAGES, UIDNEXT, and UIDVALIDITY. If MessageExpunge is enabled, the STATUS response MUST contain MESSAGES. If either AnnotationChange or FlagChange are Gulbrandsen, et al. Standards Track [Page 5] RFC 5465 IMAP NOTIFY Extension February 2009 included and the server also supports the CONDSTORE [RFC4551] and/or QRESYNC [RFC5162] extensions, the STATUS response MUST contain UIDVALIDITY and HIGHESTMODSEQ. Absence of the STATUS indicator parameter allows the client to avoid the additional STATUS responses. This might be useful if the client already retrieved this information before issuing the NOTIFY command. Clients are advised to limit the number of mailboxes used with NOTIFY. Particularly, if a client asks for events for all accessible mailboxes, the server may swamp the client with updates about shared mailboxes. This may reduce the client's battery life. Also, this wastes both server and network resources. For each mailbox specified, the server verifies that the client has access using the following test: - If the name does not refer to an existing mailbox, the server MUST ignore it. - If the name refers to a mailbox that the client can't LIST, the server MUST ignore it. For a server that implements [RFC4314], this means that if the client doesn't have the 'l' (lookup) right for the name, then the server MUST ignore the mailbox. This behavior prevents disclosure of potentially confidential information to clients who don't have rights to know it. - If the name refers to a mailbox that the client can LIST (e.g., it has the 'l' right from [RFC4314]), but the client doesn't have another right required for processing of the specified event(s), then the server MUST respond with an untagged extended LIST response containing the \NoAccess name attribute. The server SHOULD return the tagged OK response if the client has access to at least one of the mailboxes specified in the current list of interesting events. The server MAY return the tagged NO response if the client has no access to any of the specified mailboxes and no access can ever be granted in the future (e.g., the client specified an event for 'Subtree Bar/Foo', 'Bar/Foo' doesn't exist, and LIST returns \Noinferiors for the parent 'Bar'). If the notification would be prohibitively expensive for the server (e.g., "notify me of all flag changes in all mailboxes"), the server MAY refuse the command with a tagged NO [NOTIFICATIONOVERFLOW] response. Gulbrandsen, et al. Standards Track [Page 6] RFC 5465 IMAP NOTIFY Extension February 2009 If the client requests information for events of an unsupported type, the server MUST refuse the command with a tagged NO response (not a BAD). This response SHOULD contain the BADEVENT response code, which MUST list names of all events supported by the server. Here's an example: S: * OK [CAPABILITY IMAP4REV1 NOTIFY] C: a login bob alice S: a OK Password matched C: b notify set status (selected MessageNew (uid body.peek[header.fields (from to subject)]) MessageExpunge) (subtree Lists MessageNew) S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 9999 MESSAGES 500) S: [...] S: * STATUS Lists/Im2000 (UIDVALIDITY 901 UIDNEXT 1 MESSAGES 0) S: b OK done C: c select inbox S: [...] (the usual 7-8 responses to SELECT) S: c OK INBOX selected (Time passes. A new message is delivered to mailbox Lists/Lemonade.) S: * STATUS Lists/Lemonade (UIDVALIDITY 4 UIDNEXT 10000 MESSAGES 501) (Time passes. A new message is delivered to inbox.) S: * 127 FETCH (UID 127001 BODY[HEADER.FIELDS (From To Subject)] {75} S: Subject: Re: good morning S: From: alice@example.org S: To: bob@example.org S: S: ) (Time passes. The client decides it wants to know about one more mailbox. As the client already knows necessary STATUS information for all mailboxes below the Lists mailbox, and because "notify set status" would cause STATUS responses for *all* mailboxes specified in the NOTIFY command, including the ones for which the client already knows STATUS information, the client issues an explicit STATUS request for the mailbox to be added to the watch list, followed by the NOTIFY SET without the STATUS parameter.) C: d STATUS misc (UIDVALIDITY UIDNEXT MESSAGES) S: * STATUS misc (UIDVALIDITY 1 UIDNEXT 999) S: d STATUS completed Gulbrandsen, et al. Standards Track [Page 7] RFC 5465 IMAP NOTIFY Extension February 2009 C: e notify set (selected MessageNew (uid body.peek[header.fields (from to subject)]) MessageExpunge) (subtree Lists MessageNew) (mailboxes misc MessageNew) S: e OK done 4. Interaction with the IDLE Command If IDLE [RFC2177] (as well as this extension) is supported, then while processing any IDLE command, the server MUST send exactly the same events as instructed by the client using the NOTIFY command. NOTIFY makes IDLE unnecessary for some clients. If a client does not use MSNs and '*' in commands, it can request MessageExpunge and MessageNew for the selected mailbox by using the NOTIFY command instead of entering the IDLE mode. A client that uses MSNs and '*' in commands can still use the NOTIFY command if it specifies the SELECTED-DELAYED mailbox specifier in the NOTIFY command. 5. Event Types Only some of the events in [RFC5423] can be expressed in IMAP, and for some of them there are several possible ways to express the event. This section specifies the events of which an IMAP server can notify an IMAP client, and how. The server SHOULD omit notifying the client if the event is caused by this client. For example, if the client issues CREATE and has requested a MailboxName event that would cover the newly created mailbox, the server SHOULD NOT notify the client of the MailboxName change. All event types described in this document require the 'l' and 'r' rights (see [RFC4314]) on all observed mailboxes. Servers that don't implement [RFC4314] should map the above rights to their access- control model. If the FlagChange and/or AnnotationChange events are specified, MessageNew and MessageExpunge MUST also be specified by the client. Otherwise, the server MUST respond with the tagged BAD response. If one of MessageNew or MessageExpunge is specified, then both events MUST be specified. Otherwise, the server MUST respond with the tagged BAD response. Gulbrandsen, et al. Standards Track [Page 8] RFC 5465 IMAP NOTIFY Extension February 2009 The client can instruct the server not to send an event by omitting the necessary event from the list of events specified in NOTIFY SET, by using the NONE event specifier in the NOTIFY SET, or by using NOTIFY NONE. In particular, NOTIFY SET ... NONE can be used as a snapshot facility by clients. 5.1. FlagChange and AnnotationChange If the flag and/or message annotation change happens in the selected mailbox, the server MUST notify the client by sending an unsolicited FETCH response, which MUST include UID and FLAGS/ANNOTATION FETCH data items. It MAY also send new FLAGS and/or OK [PERMANENTFLAGS ...] responses. If a search context is in effect as specified in [RFC5267], an ESEARCH ADDTO or ESEARCH REMOVEFROM will also be generated, if appropriate. In this case, the FETCH response MUST precede the ESEARCH response. If the change happens in another mailbox, then the server responds with a STATUS response. The exact content of the STATUS response depends on various factors. If CONDSTORE [RFC4551] and/or QRESYNC [RFC5162] are enabled by the client, then the server sends a STATUS response that includes at least HIGHESTMODSEQ and UIDVALIDITY status data items. If the number of messages with the \Seen flag changes, the server MAY also include the UNSEEN data item in the STATUS response. If CONDSTORE/QRESYNC is not enabled by the client and the server chooses not to include the UNSEEN data item, the server does not notify the client. When this event is requested, the server MUST notify the client about mailbox UIDVALIDITY changes. This is done by sending a STATUS response that includes UIDVALIDITY. FlagChange covers the MessageRead, MessageTrash, FlagsSet, and FlagsClear events in [RFC5423]. Example in the selected mailbox: S: * 99 FETCH (UID 9999 FLAGS ($Junk)) And in another mailbox, with CONDSTORE in use: S: * STATUS Lists/Lemonade (HIGHESTMODSEQ 65666665 UIDVALIDITY 101) Gulbrandsen, et al. Standards Track [Page 9] RFC 5465 IMAP NOTIFY Extension February 2009 5.2. MessageNew This covers both MessageNew and MessageAppend in [RFC5423]. If the new/appended message is in the selected mailbox, the server notifies the client by sending an unsolicited EXISTS response, followed by an unsolicited FETCH response containing the information requested by the client. A FETCH response SHOULD NOT be generated for a new message created by the client on this particular connection, for instance, as the result of an APPEND or COPY command to the selected mailbox performed by the client itself. The server MAY also send a RECENT response, if the server marks the message as \Recent. Note that a single EXISTS response can be returned for multiple MessageAppend/MessageNew events. If a search context is in effect as specified in [RFC5267], an ESEARCH ADDTO will also be generated, if appropriate. In this case, the EXISTS response MUST precede the ESEARCH response. Both the NOTIFY command and the SEARCH and SORT commands (see Section 7) can specify attributes to be returned for new messages. These attributes SHOULD be combined into a single FETCH response. The server SHOULD avoid sending duplicate data. The FETCH response(s) MUST follow any ESEARCH ADDTO responses. If the new/appended message is in another mailbox, the server sends an unsolicited STATUS (UIDNEXT MESSAGES) response for the relevant mailbox. If the CONDSTORE extension [RFC4551] and/or the QRESYNC extension [RFC5162] is enabled, the HIGHESTMODSEQ status data item MUST be included in the STATUS response. The client SHOULD NOT use FETCH attributes that implicitly set the \seen flag, or that presuppose the existence of a given bodypart. UID, MODSEQ, FLAGS, ENVELOPE, BODY.PEEK[HEADER.FIELDS... and BODY/BODYSTRUCTURE may be the most useful attributes. Note that if a client asks to be notified of MessageNew events with the SELECTED mailbox specifier, the number of messages can increase at any time, and therefore the client cannot refer to a specific message using the MSN/UID '*'. Example in the selected mailbox: S: * 444 EXISTS S: * 444 FETCH (UID 9999) And in another mailbox, without CONDSTORE enabled: S: * STATUS Lists/Lemonade (UIDNEXT 10002 MESSAGES 503) Gulbrandsen, et al. Standards Track [Page 10] RFC 5465 IMAP NOTIFY Extension February 2009 5.3. MessageExpunge If the expunged message or messages are in the selected mailbox, the server notifies the client using EXPUNGE (or VANISHED, if [RFC5162] is supported by the server and enabled by the client). If a search context is in effect, as specified in [RFC5267], an ESEARCH REMOVEFROM will also be generated, if appropriate. If the expunged message or messages are in another mailbox, the server sends an unsolicited STATUS (UIDNEXT MESSAGES) response for the relevant mailbox. If the QRESYNC [RFC5162] extension is enabled, the HIGHESTMODSEQ data item MUST be included in the STATUS response as well. Note that if a client requests MessageExpunge with the SELECTED mailbox specifier, the meaning of an MSN can change at any time, so the client cannot use MSNs in commands anymore. For example, such a client cannot use FETCH, but has to use UID FETCH. The meaning of '*' can also change when messages are added or expunged. A client wishing to keep using MSNs can either use the SELECTED-DELAYED mailbox specifier or can avoid using the MessageExpunge event entirely. The MessageExpunge notification covers both MessageExpunge and MessageExpire events from [RFC5423]. Example in the selected mailbox, without QRESYNC: S: * 444 EXPUNGE The same example in the selected mailbox, with QRESYNC: S: * VANISHED 5444 And in another mailbox, when QRESYNC is not enabled: S: * STATUS misc (UIDNEXT 999 MESSAGES 554) 5.4. MailboxName These notifications are sent if an affected mailbox name was created (with CREATE), deleted (with DELETE), or renamed (with RENAME). For a server that implements [RFC4314], granting or revocation of the 'l' right to the current user on the affected mailbox MUST be considered mailbox creation or deletion, respectively. If a mailbox is created or deleted, the mailbox itself and its direct parent (whether it is an existing mailbox or not) are considered to be affected. Gulbrandsen, et al. Standards Track [Page 11] RFC 5465 IMAP NOTIFY Extension February 2009 The server notifies the client by sending an unsolicited LIST response for each affected mailbox name. If, after the event, the mailbox name does not refer to a mailbox accessible to the client, the \Nonexistent flag MUST be included. For each LISTable mailbox renamed, the server sends an extended LIST response [RFC5258] for the new mailbox name, containing the OLDNAME extended data item with the old mailbox name. When a mailbox is renamed, its children are renamed too. No additional MailboxName events are sent for children in this case. When INBOX is renamed, a new INBOX is assumed to be created. No MailboxName event is sent for INBOX in this case. If the server automatically subscribes a mailbox when it is created or renamed, then the unsolicited LIST response for each affected subscribed mailbox name MUST include the \Subscribed attribute (see [RFC5258]). The server SHOULD also include \HasChildren or \HasNoChildren attributes [RFC5258] as appropriate. Example of a newly created mailbox (or granting of the 'l' right on the mailbox): S: * LIST () "/" "NewMailbox" And a deleted mailbox (or revocation of the 'l' right on the mailbox): S: * LIST (\NonExistent) "." "INBOX.DeletedMailbox" Example of a renamed mailbox: S: * LIST () "/" "NewMailbox" ("OLDNAME" ("OldMailbox")) 5.5. SubscriptionChange The server notifies the client by sending an unsolicited LIST response for each affected mailbox name. If and only if the mailbox is subscribed after the event, the \Subscribed attribute (see [RFC5258]) is included. Note that in the LIST response, all mailbox attributes MUST be accurately computed (this differs from the behavior of the LSUB command). Example: S: * LIST (\Subscribed) "/" "SubscribedMailbox" 5.6. MailboxMetadataChange Support for this event type is OPTIONAL unless the METADATA extension [RFC5464] is also supported by the server, in which case support for this event type is REQUIRED. Gulbrandsen, et al. Standards Track [Page 12] RFC 5465 IMAP NOTIFY Extension February 2009 A client willing to receive unsolicited METADATA responses as a result of using the MailboxMetadataChange event in the NOTIFY command doesn't have to issue ENABLE METADATA. The server sends an unsolicited METADATA response (as per Section 4.4.2 of [RFC5464]). If possible, only the changed metadata SHOULD be included, but if the server can't detect a change to a single metadata item, it MAY include all metadata items set on the mailbox. If a metadata item is deleted (set to NIL), it MUST always be included in the METADATA response. Example: S: * METADATA "INBOX" /shared/comment 5.7. ServerMetadataChange Support for this event type is OPTIONAL unless the METADATA or the METADATA-SERVER extension [RFC5464] is also supported by the server, in which case support for this event type is REQUIRED. A client willing to receive unsolicited METADATA responses as a result of using the ServerMetadataChange event in the NOTIFY command doesn't have to issue ENABLE METADATA or ENABLE METADATA-SERVER. The server sends an unsolicited METADATA response (as per Section 4.4.2 of [RFC5464]). Only the names of changed metadata entries SHOULD be returned in such METADATA responses. If a metadata item is deleted (set to NIL), it MUST always be included in the METADATA response. Example: S: * METADATA "" /shared/comment 5.8. Notification Overflow If the server is unable or unwilling to deliver as many notifications as it is being asked to, it may disable notifications for some or all clients. It MUST notify these clients by sending an untagged "OK [NOTIFICATIONOVERFLOW]" response and behave as if a NOTIFY NONE command had just been received. Example: S: * OK [NOTIFICATIONOVERFLOW] ...A comment can go here... Gulbrandsen, et al. Standards Track [Page 13] RFC 5465 IMAP NOTIFY Extension February 2009 5.9. ACL (Access Control List) Changes Even if NOTIFY succeeds, it is still possible to lose access to the mailboxes being monitored at a later time. If this happens, the server MUST stop monitoring these mailboxes. If access is later granted, the server MUST restart event monitoring. The server SHOULD return the LIST response with the \NoAccess name attribute if and when the mailbox loses the 'l' right. Similarly, the server SHOULD return the LIST response with no \NoAccess name attribute if the mailbox was previously reported as having \NoAccess and the 'l' right is later granted. 6. Mailbox Specification Mailboxes to be monitored can be specified in several different ways. Only 'SELECTED' and 'SELECTED-DELAYED' (Section 6.1) match the currently selected mailbox. All other mailbox specifications affect other (non-selected) mailboxes. Note that multiple s can apply to the same mailbox. The following example demonstrates this. In this example, MessageNew and MessageExpunge events are reported for INBOX, due to the first . A SubscriptionChange event will also be reported for INBOX, due to the second . C: a notify set (mailboxes INBOX (Messagenew messageExpunge)) (personal (SubscriptionChange)) A typical client that supports the NOTIFY extension would ask for events on the selected mailbox and some named mailboxes. In the next example, the client asks for FlagChange events for all personal mailboxes except the currently selected mailbox. This is different from the previous example because SELECTED overrides all other message event definitions for the currently selected mailbox (see Section 3.1). C: a notify set (selected (Messagenew (uid flags) messageExpunge)) (personal (MessageNew FlagChange MessageExpunge)) 6.1. Mailbox Specifiers Affecting the Currently Selected Mailbox Only one of the mailbox specifiers affecting the currently selected mailbox can be specified in any NOTIFY command. The two such mailbox specifiers (SELECTED and SELECTED-DELAYED) are described below. Gulbrandsen, et al. Standards Track [Page 14] RFC 5465 IMAP NOTIFY Extension February 2009 Both refer to the mailbox that was selected using either SELECT or EXAMINE (see [RFC3501], Sections 6.3.1 and 6.3.2). When the IMAP connection is not in the selected state, such mailbox specifiers don't refer to any mailbox. The mailbox specifiers only apply to s. It is an error to specify other types of events with either the SELECTED or the SELECTED-DELAYED selector. 6.1.1. Selected The SELECTED mailbox specifier requires the server to send immediate notifications for the currently selected mailbox about all specified s. 6.1.2. Selected-Delayed The SELECTED-DELAYED mailbox specifier requires the server to delay a MessageExpunge event until the client issues a command that allows returning information about expunged messages (see Section 7.4.1 of [RFC3501] for more details), for example, till a NOOP or an IDLE command has been issued. When SELECTED-DELAYED is specified, the server MAY also delay returning other s until the client issues one of the commands specified above, or it MAY return them immediately. 6.2. Personal Personal refers to all selectable mailboxes in the user's personal namespace(s), as defined in [RFC2342]. 6.3. Inboxes Inboxes refers to all selectable mailboxes in the user's personal namespace(s) to which messages may be delivered by a Message Delivery Agent (MDA) (see [EMAIL-ARCH], particularly Section 4.3.3). If the IMAP server cannot easily compute this set, it MUST treat "inboxes" as equivalent to "personal". 6.4. Subscribed Subscribed refers to all mailboxes subscribed to by the user. If the subscription list changes, the server MUST reevaluate the list. Gulbrandsen, et al. Standards Track [Page 15] RFC 5465 IMAP NOTIFY Extension February 2009 6.5. Subtree Subtree is followed by a mailbox name or list of mailbox names. A subtree refers to all selectable mailboxes that are subordinate to the specified mailbox plus the specified mailbox itself. 6.6. Mailboxes Mailboxes is followed by a mailbox name or a list of mailbox names. The server MUST NOT do a wildcard expansion. This means there is no special treatment for the LIST wildcard characters ('*' and '%') if they are present in mailbox names. 7. Extension to SEARCH and SORT Commands If the server that supports the NOTIFY extension also supports CONTEXT=SEARCH and/or CONTEXT=SORT as defined in [RFC5267], the UPDATE return option is extended so that a client can request that FETCH attributes be returned when a new message is added to the context result set. For example: C: a00 SEARCH RETURN (COUNT UPDATE (UID BODY[HEADER.FIELDS (TO FROM SUBJECT)])) FROM "boss" S: * ESEARCH (TAG "a00") (COUNT 17) S: a00 OK [...a new message is delivered...] S: * EXISTS 93 S: * 93 FETCH (UID 127001 BODY[HEADER.FIELDS (FROM TO SUBJECT)] {76} S: Subject: Re: good morning S: From: myboss@example.org S: To: bob@example.org S: S: ) S: * ESEARCH (TAG "a00") ADDTO (0 93) Note that the EXISTS response MUST precede any FETCH responses, and together they MUST precede the ESEARCH response. No untagged FETCH response SHOULD be returned if a message becomes a member of UPDATE SEARCH due to flag or annotation changes. Gulbrandsen, et al. Standards Track [Page 16] RFC 5465 IMAP NOTIFY Extension February 2009 8. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines the non-terminals "capability", "command-auth", "mailbox", "mailbox- data", "resp-text-code", and "search-key". The "modifier-update" non-terminal is defined in [RFC5267]. "mbx-list-oflag" is defined in [RFC3501] and updated by [RFC5258]. Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lower case characters to define token strings is for editorial clarity only. Implementations MUST accept these strings in a case-insensitive fashion. For example, the non-terminal value "SELECTED" must be treated in the same way as "Selected" or "selected". capability =/ "NOTIFY" command-auth =/ notify notify = "NOTIFY" SP (notify-set / notify-none) notify-set = "SET" [status-indicator] SP event-groups ; Replace registered notification events ; with the specified list of events notify-none = "NONE" ; Cancel all registered notification ; events. The client is not interested ; in receiving any events. status-indicator = SP "STATUS" one-or-more-mailbox = mailbox / many-mailboxes many-mailboxes = "(" mailbox *(SP mailbox) ")" event-groups = event-group *(SP event-group) event-group = "(" filter-mailboxes SP events ")" ;; Only s are allowed in ;; when is used. filter-mailboxes = filter-mailboxes-selected / filter-mailboxes-other Gulbrandsen, et al. Standards Track [Page 17] RFC 5465 IMAP NOTIFY Extension February 2009 filter-mailboxes-other = "inboxes" / "personal" / "subscribed" / ( "subtree" SP one-or-more-mailbox ) / ( "mailboxes" SP one-or-more-mailbox ) filter-mailboxes-selected = "selected" / "selected-delayed" ;; Apply to the currently selected mailbox only. ;; Only one of them can be specified in a NOTIFY ;; command. events = ( "(" event *(SP event) ")" ) / "NONE" ;; As in [MSGEVENT]. ;; "NONE" means that the client does not wish ;; to receive any events for the specified ;; mailboxes. event = message-event / mailbox-event / user-event / event-ext message-event = ( "MessageNew" [SP "(" fetch-att *(SP fetch-att) ")" ] ) / "MessageExpunge" / "FlagChange" / "AnnotationChange" ;; "MessageNew" includes "MessageAppend" from ;; [MSGEVENT]. "FlagChange" is any of ;; "MessageRead", "MessageTrash", "FlagsSet", ;; "FlagsClear" [MSGEVENT]. "MessageExpunge" ;; includes "MessageExpire" [MSGEVENT]. ;; MessageNew and MessageExpunge MUST always ;; be specified together. If FlagChange is ;; specified, then MessageNew and MessageExpunge ;; MUST be specified as well. ;; The fett-att list may only be present for the ;; SELECTED/SELECTED-DELAYED mailbox filter ;; (). mailbox-event = "MailboxName" / "SubscriptionChange" / "MailboxMetadataChange" ; "SubscriptionChange" includes ; MailboxSubscribe and MailboxUnSubscribe. ; "MailboxName" includes MailboxCreate, ; "MailboxDelete" and "MailboxRename". user-event = "ServerMetadataChange" event-ext = atom ;; For future extensions Gulbrandsen, et al. Standards Track [Page 18] RFC 5465 IMAP NOTIFY Extension February 2009 oldname-extended-item = "OLDNAME" SP "(" mailbox ")" ;; Extended data item (mbox-list-extended-item) ;; returned in a LIST response when a mailbox is ;; renamed. ;; Note 1: the OLDNAME tag can be returned ;; with or without surrounding quotes, as per ;; mbox-list-extended-item-tag production. resp-text-code =/ "NOTIFICATIONOVERFLOW" / unsupported-events-code message-event-name = "MessageNew" / "MessageExpunge" / "FlagChange" / "AnnotationChange" event-name = message-event-name / mailbox-event / user-event unsupported-events-code = "BADEVENT" SP "(" event-name *(SP event-name) ")" modifier-update = "UPDATE" [ "(" fetch-att *(SP fetch-att) ")" ] mbx-list-oflag =/ "\NoAccess" 9. Security Considerations It is very easy for a client to deny itself service using NOTIFY. Asking for all events on all mailboxes may work on a small server, but with a big server, can swamp the client's network connection or processing capability. In the worst case, the server's processing could also degrade the service it offers to other clients. Server authors should be aware that if a client issues requests and does not listen to the resulting responses, the TCP window can easily fill up, and a careless server might block. This problem also exists in plain IMAP; however, this extension magnifies the problem. This extension makes it possible to retrieve messages immediately when they are added to the mailbox. This makes it wholly impractical to delete sensitive messages using programs like imapfilter. Using SIEVE [RFC5228] or similar is much better. Gulbrandsen, et al. Standards Track [Page 19] RFC 5465 IMAP NOTIFY Extension February 2009 10. IANA Considerations The IANA has added NOTIFY to the list of IMAP extensions. 10.1. Initial LIST-EXTENDED Extended Data Item Registrations The following entry has been added to the LIST-EXTENDED response registry [RFC5258]: To: iana@iana.org Subject: Registration of OLDNAME LIST-EXTENDED extended data item LIST-EXTENDED extended data item tag: OLDNAME LIST-EXTENDED extended data item description: The OLDNAME extended data item describes the old mailbox name for the mailbox identified by the LIST response. Which LIST-EXTENDED option(s) (and their types) causes this extended data item to be returned (if any): none Published specification : RFC 5465, Section 5.4. Security considerations: none Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 11. Acknowledgments The authors gratefully acknowledge the help of Peter Coates, Dave Cridland, Mark Crispin, Cyrus Daboo, Abhijit Menon-Sen, Timo Sirainen, and Eric Burger. In particular, Peter Coates contributed lots of text and useful suggestions to this document. Various examples are copied from other RFCs. This document builds on one published and two unpublished drafts by the same authors. Gulbrandsen, et al. Standards Track [Page 20] RFC 5465 IMAP NOTIFY Extension February 2009 12. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC2177] Leiba, B., "IMAP4 IDLE command", RFC 2177, June 1997. [RFC2342] Gahrns, M. and C. Newman, "IMAP4 Namespace", RFC 2342, May 1998. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC4551] Melnikov, A. and S. Hole, "IMAP Extension for Conditional STORE Operation or Quick Flag Changes Resynchronization", RFC 4551, June 2006. [RFC5162] Melnikov, A., Cridland, D., and C. Wilson, "IMAP4 Extensions for Quick Mailbox Resynchronization", RFC 5162, March 2008. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access Protocol version 4 - LIST Command Extensions", RFC 5258, June 2008. [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC 5267, July 2008. [RFC5423] Newman, C. and R. Gellens, "Internet Message Store Events", RFC 5423, Month 2009. [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, February 2009. 13. Informative References [RFC5228] Guenther, P., Ed., and T. Showalter, Ed., "Sieve: An Email Filtering Language", RFC 5228, January 2008. Gulbrandsen, et al. Standards Track [Page 21] RFC 5465 IMAP NOTIFY Extension February 2009 [EMAIL-ARCH] Crocker, D., "Internet Mail Architecture", Work in Progress, October 2008. Authors' Addresses Arnt Gulbrandsen Oryx Mail Systems GmbH Schweppermannstr. 8 D-81671 Muenchen Germany EMail: arnt@oryx.com Curtis King Isode Ltd 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Curtis.King@isode.com Alexey Melnikov Isode Ltd 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com Gulbrandsen, et al. Standards Track [Page 22] ================================================ FILE: docs/rfcs/rfc5530.IMAP_Response_codes.txt ================================================ Network Working Group A. Gulbrandsen Request for Comments: 5530 Oryx Mail Systems GmbH Category: Standards Track May 2009 IMAP Response Codes Status of This Memo This document specifies an Internet standards track protocol for the Internet community, and requests discussion and suggestions for improvements. Please refer to the current edition of the "Internet Official Protocol Standards" (STD 1) for the standardization state and status of this protocol. Distribution of this memo is unlimited. Copyright Notice Copyright (c) 2009 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents in effect on the date of publication of this document (http://trustee.ietf.org/license-info). Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Abstract IMAP responses consist of a response type (OK, NO, BAD), an optional machine-readable response code, and a human-readable text. This document collects and documents a variety of machine-readable response codes, for better interoperation and error reporting. Gulbrandsen Standards Track [Page 1] RFC 5530 IMAP Response Codes May 2009 1. Introduction Section 7.1 of [RFC3501] defines a number of response codes that can help tell an IMAP client why a command failed. However, experience has shown that more codes are useful. For example, it is useful for a client to know that an authentication attempt failed because of a server problem as opposed to a password problem. Currently, many IMAP servers use English-language, human-readable text to describe these errors, and a few IMAP clients attempt to translate this text into the user's language. This document names a variety of errors as response codes. It is based on errors that have been checked and reported on in some IMAP server implementations, and on the needs of some IMAP clients. This document doesn't require any servers to test for these errors or any clients to test for these names. It only names errors for better reporting and handling. 2. Conventions Used in This Document Formal syntax is defined by [RFC5234] as modified by [RFC3501]. Example lines prefaced by "C:" are sent by the client and ones prefaced by "S:" by the server. "[...]" means elision. 3. Response Codes This section defines all the new response codes. Each definition is followed by one or more examples. UNAVAILABLE Temporary failure because a subsystem is down. For example, an IMAP server that uses a Lightweight Directory Access Protocol (LDAP) or Radius server for authentication might use this response code when the LDAP/Radius server is down. C: a LOGIN "fred" "foo" S: a NO [UNAVAILABLE] User's backend down for maintenance AUTHENTICATIONFAILED Authentication failed for some reason on which the server is unwilling to elaborate. Typically, this includes "unknown user" and "bad password". Gulbrandsen Standards Track [Page 2] RFC 5530 IMAP Response Codes May 2009 This is the same as not sending any response code, except that when a client sees AUTHENTICATIONFAILED, it knows that the problem wasn't, e.g., UNAVAILABLE, so there's no point in trying the same login/password again later. C: b LOGIN "fred" "foo" S: b NO [AUTHENTICATIONFAILED] Authentication failed AUTHORIZATIONFAILED Authentication succeeded in using the authentication identity, but the server cannot or will not allow the authentication identity to act as the requested authorization identity. This is only applicable when the authentication and authorization identities are different. C: c1 AUTHENTICATE PLAIN [...] S: c1 NO [AUTHORIZATIONFAILED] No such authorization-ID C: c2 AUTHENTICATE PLAIN [...] S: c2 NO [AUTHORIZATIONFAILED] Authenticator is not an admin EXPIRED Either authentication succeeded or the server no longer had the necessary data; either way, access is no longer permitted using that passphrase. The client or user should get a new passphrase. C: d login "fred" "foo" S: d NO [EXPIRED] That password isn't valid any more PRIVACYREQUIRED The operation is not permitted due to a lack of privacy. If Transport Layer Security (TLS) is not in use, the client could try STARTTLS (see Section 6.2.1 of [RFC3501]) and then repeat the operation. C: d login "fred" "foo" S: d NO [PRIVACYREQUIRED] Connection offers no privacy C: d select inbox S: d NO [PRIVACYREQUIRED] Connection offers no privacy Gulbrandsen Standards Track [Page 3] RFC 5530 IMAP Response Codes May 2009 CONTACTADMIN The user should contact the system administrator or support desk. C: e login "fred" "foo" S: e OK [CONTACTADMIN] NOPERM The access control system (e.g., Access Control List (ACL), see [RFC4314]) does not permit this user to carry out an operation, such as selecting or creating a mailbox. C: f select "/archive/projects/experiment-iv" S: f NO [NOPERM] Access denied INUSE An operation has not been carried out because it involves sawing off a branch someone else is sitting on. Someone else may be holding an exclusive lock needed for this operation, or the operation may involve deleting a resource someone else is using, typically a mailbox. The operation may succeed if the client tries again later. C: g delete "/archive/projects/experiment-iv" S: g NO [INUSE] Mailbox in use EXPUNGEISSUED Someone else has issued an EXPUNGE for the same mailbox. The client may want to issue NOOP soon. [RFC2180] discusses this subject in depth. C: h search from fred@example.com S: * SEARCH 1 2 3 5 8 13 21 42 S: h OK [EXPUNGEISSUED] Search completed CORRUPTION The server discovered that some relevant data (e.g., the mailbox) are corrupt. This response code does not include any information about what's corrupt, but the server can write that to its logfiles. C: i select "/archive/projects/experiment-iv" S: i NO [CORRUPTION] Cannot open mailbox Gulbrandsen Standards Track [Page 4] RFC 5530 IMAP Response Codes May 2009 SERVERBUG The server encountered a bug in itself or violated one of its own invariants. C: j select "/archive/projects/experiment-iv" S: j NO [SERVERBUG] This should not happen CLIENTBUG The server has detected a client bug. This can accompany all of OK, NO, and BAD, depending on what the client bug is. C: k1 select "/archive/projects/experiment-iv" [...] S: k1 OK [READ-ONLY] Done C: k2 status "/archive/projects/experiment-iv" (messages) [...] S: k2 OK [CLIENTBUG] Done CANNOT The operation violates some invariant of the server and can never succeed. C: l create "///////" S: l NO [CANNOT] Adjacent slashes are not supported LIMIT The operation ran up against an implementation limit of some kind, such as the number of flags on a single message or the number of flags used in a mailbox. C: m STORE 42 FLAGS f1 f2 f3 f4 f5 ... f250 S: m NO [LIMIT] At most 32 flags in one mailbox supported OVERQUOTA The user would be over quota after the operation. (The user may or may not be over quota already.) Note that if the server sends OVERQUOTA but doesn't support the IMAP QUOTA extension defined by [RFC2087], then there is a quota, but the client cannot find out what the quota is. C: n1 uid copy 1:* oldmail S: n1 NO [OVERQUOTA] Sorry C: n2 uid copy 1:* oldmail S: n2 OK [OVERQUOTA] You are now over your soft quota Gulbrandsen Standards Track [Page 5] RFC 5530 IMAP Response Codes May 2009 ALREADYEXISTS The operation attempts to create something that already exists, such as when the CREATE or RENAME directories attempt to create a mailbox and there is already one of that name. C: o RENAME this that S: o NO [ALREADYEXISTS] Mailbox "that" already exists NONEXISTENT The operation attempts to delete something that does not exist. Similar to ALREADYEXISTS. C: p RENAME this that S: p NO [NONEXISTENT] No such mailbox 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234]. [RFC3501] defines the non-terminal "resp-text-code". Except as noted otherwise, all alphabetic characters are case- insensitive. The use of upper or lowercase characters to define token strings is for editorial clarity only. resp-text-code =/ "UNAVAILABLE" / "AUTHENTICATIONFAILED" / "AUTHORIZATIONFAILED" / "EXPIRED" / "PRIVACYREQUIRED" / "CONTACTADMIN" / "NOPERM" / "INUSE" / "EXPUNGEISSUED" / "CORRUPTION" / "SERVERBUG" / "CLIENTBUG" / "CANNOT" / "LIMIT" / "OVERQUOTA" / "ALREADYEXISTS" / "NONEXISTENT" 5. Security Considerations Revealing information about a passphrase to unauthenticated IMAP clients causes bad karma. Response codes are easier to parse than human-readable text. This can amplify the consequences of an information leak. For example, selecting a mailbox can fail because the mailbox doesn't exist, because the user doesn't have the "l" right (right to know the mailbox exists) or "r" right (right to read the mailbox). If the server sent different responses in the first two cases in the past, only malevolent clients would discover it. With response codes it's possible, perhaps probable, that benevolent clients will forward the Gulbrandsen Standards Track [Page 6] RFC 5530 IMAP Response Codes May 2009 leaked information to the user. Server authors are encouraged to be particularly careful with the NOPERM and authentication-related responses. 6. IANA Considerations The IANA has created the IMAP Response Codes registry. The registry has been populated with the following codes: NEWNAME RFC 2060 (obsolete) REFERRAL RFC 2221 ALERT RFC 3501 BADCHARSET RFC 3501 PARSE RFC 3501 PERMANENTFLAGS RFC 3501 READ-ONLY RFC 3501 READ-WRITE RFC 3501 TRYCREATE RFC 3501 UIDNEXT RFC 3501 UIDVALIDITY RFC 3501 UNSEEN RFC 3501 UNKNOWN-CTE RFC 3516 UIDNOTSTICKY RFC 4315 APPENDUID RFC 4315 COPYUID RFC 4315 URLMECH RFC 4467 TOOBIG RFC 4469 BADURL RFC 4469 HIGHESTMODSEQ RFC 4551 NOMODSEQ RFC 4551 MODIFIED RFC 4551 COMPRESSIONACTIVE RFC 4978 CLOSED RFC 5162 NOTSAVED RFC 5182 BADCOMPARATOR RFC 5255 ANNOTATE RFC 5257 ANNOTATIONS RFC 5257 TEMPFAIL RFC 5259 MAXCONVERTMESSAGES RFC 5259 MAXCONVERTPARTS RFC 5259 NOUPDATE RFC 5267 METADATA RFC 5464 NOTIFICATIONOVERFLOW RFC 5465 BADEVENT RFC 5465 UNDEFINED-FILTER RFC 5466 UNAVAILABLE RFC 5530 AUTHENTICATIONFAILED RFC 5530 AUTHORIZATIONFAILED RFC 5530 Gulbrandsen Standards Track [Page 7] RFC 5530 IMAP Response Codes May 2009 EXPIRED RFC 5530 PRIVACYREQUIRED RFC 5530 CONTACTADMIN RFC 5530 NOPERM RFC 5530 INUSE RFC 5530 EXPUNGEISSUED RFC 5530 CORRUPTION RFC 5530 SERVERBUG RFC 5530 CLIENTBUG RFC 5530 CANNOT RFC 5530 LIMIT RFC 5530 OVERQUOTA RFC 5530 ALREADYEXISTS RFC 5530 NONEXISTENT RFC 5530 The new registry can be extended by sending a registration request to IANA. IANA will forward this request to a Designated Expert, appointed by the responsible IESG Area Director, CCing it to the IMAP Extensions mailing list at (or a successor designated by the Area Director). After either allowing 30 days for community input on the IMAP Extensions mailing list or a successful IETF Last Call, the expert will determine the appropriateness of the registration request and either approve or disapprove the request by sending a notice of the decision to the requestor, CCing the IMAP Extensions mailing list and IANA. A denial notice must be justified by an explanation, and, in cases where it is possible, concrete suggestions on how the request can be modified so as to become acceptable should be provided. For each response code, the registry contains a list of relevant RFCs that describe (or extend) the response code and an optional response code status description, such as "obsolete" or "reserved to prevent collision with deployed software". (Note that in the latter case, the RFC number can be missing.) Presence of the response code status description means that the corresponding response code is NOT RECOMMENDED for widespread use. The intention is that any future allocation will be accompanied by a published RFC (including direct submissions to the RFC Editor). But in order to allow for the allocation of values prior to the RFC being approved for publication, the Designated Expert can approve allocations once it seems clear that an RFC will be published, for example, before requesting IETF LC for the document. The Designated Expert can also approve registrations for response codes used in deployed software when no RFC exists. Such registrations must be marked as "reserved to prevent collision with deployed software". Gulbrandsen Standards Track [Page 8] RFC 5530 IMAP Response Codes May 2009 Response code registrations may not be deleted; response codes that are no longer believed appropriate for use (for example, if there is a problem with the syntax of said response code or if the specification describing it was moved to Historic) should be marked "obsolete" in the registry, clearly marking the lists published by IANA. 7. Acknowledgements Peter Coates, Mark Crispin, Philip Guenther, Alexey Melnikov, Ken Murchison, Chris Newman, Timo Sirainen, Philip Van Hoof, Dale Wiggins, and Sarah Wilkin helped with this document. 8. References 8.1. Normative References [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. 9. Informative References [RFC2087] Myers, J., "IMAP4 QUOTA extension", RFC 2087, January 1997. [RFC2180] Gahrns, M., "IMAP4 Multi-Accessed Mailbox Practice", RFC 2180, July 1997. [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. Author's Address Arnt Gulbrandsen Oryx Mail Systems GmbH Schweppermannstr. 8 D-81671 Muenchen Germany Fax: +49 89 4502 9758 EMail: arnt@oryx.com Gulbrandsen Standards Track [Page 9] ================================================ FILE: docs/rfcs/rfc5738.IMAP_UTF8.txt ================================================ Network Working Group P. Resnick Request for Comments: 5738 Qualcomm Incorporated Updates: 3501 C. Newman Category: Experimental Sun Microsystems March 2010 IMAP Support for UTF-8 Abstract This specification extends the Internet Message Access Protocol version 4rev1 (IMAP4rev1) to support UTF-8 encoded international characters in user names, mail addresses, and message headers. Status of This Memo This document is not an Internet Standards Track specification; it is published for examination, experimental implementation, and evaluation. This document defines an Experimental Protocol for the Internet community. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Not all documents approved by the IESG are a candidate for any level of Internet Standard; see Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc5738. Copyright Notice Copyright (c) 2010 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Resnick & Newman Experimental [Page 1] RFC 5738 IMAP Support for UTF-8 March 2010 This document may contain material from IETF Documents or IETF Contributions published or made publicly available before November 10, 2008. The person(s) controlling the copyright in some of this material may not have granted the IETF Trust the right to allow modifications of such material outside the IETF Standards Process. Without obtaining an adequate license from the person(s) controlling the copyright in such materials, this document may not be modified outside the IETF Standards Process, and derivative works of it may not be created outside the IETF Standards Process, except to format it for publication as an RFC or to translate it into languages other than English. Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 3. UTF8=ACCEPT IMAP Capability . . . . . . . . . . . . . . . . . 3 3.1. IMAP UTF-8 Quoted Strings . . . . . . . . . . . . . . . . 3 3.2. UTF8 Parameter to SELECT and EXAMINE . . . . . . . . . . . 5 3.3. UTF-8 LIST and LSUB Responses . . . . . . . . . . . . . . 5 3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions . . . 6 3.4.1. UTF8 and UTF8ONLY LIST Selection Options . . . . . . . 6 3.4.2. UTF8 LIST Return Option . . . . . . . . . . . . . . . 6 4. UTF8=APPEND Capability . . . . . . . . . . . . . . . . . . . . 7 5. UTF8=USER Capability . . . . . . . . . . . . . . . . . . . . . 7 6. UTF8=ALL Capability . . . . . . . . . . . . . . . . . . . . . 7 7. UTF8=ONLY Capability . . . . . . . . . . . . . . . . . . . . . 8 8. Up-Conversion Server Requirements . . . . . . . . . . . . . . 8 9. Issues with UTF-8 Header Mailstore . . . . . . . . . . . . . . 9 10. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 9 11. Security Considerations . . . . . . . . . . . . . . . . . . . 11 12. References . . . . . . . . . . . . . . . . . . . . . . . . . . 11 12.1. Normative References . . . . . . . . . . . . . . . . . . . 11 12.2. Informative References . . . . . . . . . . . . . . . . . . 13 Appendix A. Design Rationale . . . . . . . . . . . . . . . . . . 14 Appendix B. Examples Demonstrating Relationships between UTF8= Capabilities . . . . . . . . . . . . . . . . . 15 Appendix C. Acknowledgments . . . . . . . . . . . . . . . . . . . 15 Resnick & Newman Experimental [Page 2] RFC 5738 IMAP Support for UTF-8 March 2010 1. Introduction This specification extends IMAP4rev1 [RFC3501] to permit UTF-8 [RFC3629] in headers as described in "Internationalized Email Headers" [RFC5335]. It also adds a mechanism to support mailbox names, login names, and passwords using the UTF-8 charset. This specification creates five new IMAP capabilities to allow servers to advertise these new extensions, along with two new IMAP LIST selection options and a new IMAP LIST return option. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "SHOULD", "SHOULD NOT", and "MAY" in this document are to be interpreted as defined in "Key words for use in RFCs to Indicate Requirement Levels" [RFC2119]. The formal syntax uses the Augmented Backus-Naur Form (ABNF) [RFC5234] notation including the core rules defined in Appendix B of [RFC5234]. In addition, rules from IMAP4rev1 [RFC3501], UTF-8 [RFC3629], "Collected Extensions to IMAP4 ABNF" [RFC4466], and IMAP4 LIST Command Extensions [RFC5258] are also referenced. In examples, "C:" and "S:" indicate lines sent by the client and server, respectively. If a single "C:" or "S:" label applies to multiple lines, then the line breaks between those lines are for editorial clarity only and are not part of the actual protocol exchange. 3. UTF8=ACCEPT IMAP Capability The "UTF8=ACCEPT" capability indicates that the server supports UTF-8 quoted strings, the "UTF8" parameter to SELECT and EXAMINE, and UTF-8 responses from the LIST and LSUB commands. A client MUST use the "ENABLE UTF8=ACCEPT" command (defined in [RFC5161]) to indicate to the server that the client accepts UTF-8 quoted-strings. The "ENABLE UTF8=ACCEPT" command MUST only be used in the authenticated state. (Note that the "UTF8=ONLY" capability described in Section 7 and the "UTF8=ALL" capability described in Section 6 imply the "UTF8=ACCEPT" capability. See additional information in these sections.) 3.1. IMAP UTF-8 Quoted Strings The IMAP4rev1 [RFC3501] base specification forbids the use of 8-bit characters in atoms or quoted strings. Thus, a UTF-8 string can only be sent as a literal. This can be inconvenient from a coding standpoint, and unless the server offers IMAP4 non-synchronizing Resnick & Newman Experimental [Page 3] RFC 5738 IMAP Support for UTF-8 March 2010 literals [RFC2088], this requires an extra round trip for each UTF-8 string sent by the client. When the IMAP server advertises the "UTF8=ACCEPT" capability, it informs the client that it supports native UTF-8 quoted-strings with the following syntax: string =/ utf8-quoted utf8-quoted = "*" DQUOTE *UQUOTED-CHAR DQUOTE UQUOTED-CHAR = QUOTED-CHAR / UTF8-2 / UTF8-3 / UTF8-4 ; UTF8-2, UTF8-3, and UTF8-4 are as defined in RFC 3629 When this quoting mechanism is used by the client (specifically an octet sequence beginning with *" and ending with "), then the server MUST reject octet sequences with the high bit set that fail to comply with the formal syntax in [RFC3629] with a BAD response. The IMAP server MUST NOT send utf8-quoted syntax to the client unless the client has indicated support for that syntax by using the "ENABLE UTF8=ACCEPT" command. If the server advertises the "UTF8=ACCEPT" capability, the client MAY use utf8-quoted syntax with any IMAP argument that permits a string (including astring and nstring). However, if characters outside the US-ASCII repertoire are used in an inappropriate place, the results would be the same as if other syntactically valid but semantically invalid characters were used. For example, if the client includes UTF-8 characters in the user or password arguments (and the server has not advertised "UTF8=USER"), the LOGIN command will fail as it would with any other invalid user name or password. Specific cases where UTF-8 characters are permitted or not permitted are described in the following paragraphs. All IMAP servers that advertise the "UTF8=ACCEPT" capability SHOULD accept UTF-8 in mailbox names, and those that also support the "Mailbox International Naming Convention" described in RFC 3501, Section 5.1.3 MUST accept utf8-quoted mailbox names and convert them to the appropriate internal format. Mailbox names MUST comply with the Net-Unicode Definition (Section 2 of [RFC5198]) with the specific exception that they MUST NOT contain control characters (0000-001F, 0080-009F), delete (007F), line separator (2028), or paragraph separator (2029). An IMAP client MUST NOT issue a SEARCH command that uses a mixture of utf8-quoted syntax and a SEARCH CHARSET other than UTF-8. If an IMAP server receives such a SEARCH command, it SHOULD reject the command with a BAD response (due to the conflicting charset labels). Resnick & Newman Experimental [Page 4] RFC 5738 IMAP Support for UTF-8 March 2010 3.2. UTF8 Parameter to SELECT and EXAMINE The "UTF8=ACCEPT" capability also indicates that the server supports the "UTF8" parameter to SELECT and EXAMINE. When a mailbox is selected with the "UTF8" parameter, it alters the behavior of all IMAP commands related to message sizes, message headers, and MIME body headers so they refer to the message with UTF-8 headers. If the mailstore is not UTF-8 header native and the SELECT or EXAMINE command with UTF-8 header modifier succeeds, then the server MUST return results as if the mailstore were UTF-8 header native with upconversion requirements as described in Section 8. The server MAY reject the SELECT or EXAMINE command with the [NOT-UTF-8] response code, unless the "UTF8=ALL" or "UTF8=ONLY" capability is advertised. Servers MAY include mailboxes that can only be selected or examined if the "UTF8" parameter is provided. However, such mailboxes MUST NOT be included in the output of an unextended LIST, LSUB, or equivalent command. If a client attempts to SELECT or EXAMINE such mailboxes without the "UTF8" parameter, the server MUST reject the command with a [UTF-8-ONLY] response code. As a result, such mailboxes will not be accessible by IMAP clients written prior to this specification and are discouraged unless the server advertises "UTF8=ONLY" or the server implements IMAP4 LIST Command Extensions [RFC5258]. utf8-select-param = "UTF8" ;; Conforms to from RFC 4466 C: a SELECT newmailbox (UTF8) S: ... S: a OK SELECT completed C: b FETCH 1 (SIZE ENVELOPE BODY) S: ... < UTF-8 header native results > S: b OK FETCH completed C: c EXAMINE legacymailbox (UTF8) S: c NO [NOT-UTF-8] Mailbox does not support UTF-8 access C: d SELECT funky-new-mailbox S: d NO [UTF-8-ONLY] Mailbox requires UTF-8 client 3.3. UTF-8 LIST and LSUB Responses After an IMAP client successfully issues an "ENABLE UTF8=ACCEPT" command, the server MUST NOT return in LIST results any mailbox names to the client following the IMAP4 Mailbox International Naming Convention. Instead, the server MUST return any mailbox names with characters outside the US-ASCII repertoire using utf8-quoted syntax. Resnick & Newman Experimental [Page 5] RFC 5738 IMAP Support for UTF-8 March 2010 (The IMAP4 Mailbox International Naming Convention has proved problematic in the past, so the desire is to make this syntax obsolete as quickly as possible.) 3.4. UTF-8 Interaction with IMAP4 LIST Command Extensions When an IMAP server advertises both the "UTF8=ACCEPT" capability and the "LIST-EXTENDED" [RFC5258] capability, the server MUST support the LIST extensions described in this section. 3.4.1. UTF8 and UTF8ONLY LIST Selection Options The "UTF8" LIST selection option tells the server to include mailboxes that only support UTF-8 headers in the output of the list command. The "UTF8ONLY" LIST selection option tells the server to include all mailboxes that support UTF-8 headers and to exclude mailboxes that don't support UTF-8 headers. Note that "UTF8ONLY" implies "UTF8", so it is not necessary for the client to request both. Use of either selection option will also result in UTF-8 mailbox names in the result as described in Section 3.3 and implies the "UTF8" List return option described in Section 3.4.2. 3.4.2. UTF8 LIST Return Option If the client supplies the "UTF8" LIST return option, then the server MUST include either the "\NoUTF8" or the "\UTF8Only" mailbox attribute as appropriate. The "\NoUTF8" mailbox attribute indicates that an attempt to SELECT or EXAMINE that mailbox with the "UTF8" parameter will fail with a [NOT-UTF-8] response code. The "\UTF8Only" mailbox attribute indicates that an attempt to SELECT or EXAMINE that mailbox without the "UTF8" parameter will fail with a [UTF-8-ONLY] response code. Note that computing this information may be expensive on some server implementations, so this return option should not be used unless necessary. The ABNF [RFC5234] for these LIST extensions follows: list-select-independent-opt =/ "UTF8" list-select-base-opt =/ "UTF8ONLY" mbx-list-oflag =/ "\NoUTF8" / "\UTF8Only" return-option =/ "UTF8" resp-text-code =/ "NOT-UTF-8" / "UTF-8-ONLY" Resnick & Newman Experimental [Page 6] RFC 5738 IMAP Support for UTF-8 March 2010 4. UTF8=APPEND Capability If the "UTF8=APPEND" capability is advertised, then the server accepts UTF-8 headers in the APPEND command message argument. A client that sends a message with UTF-8 headers to the server MUST send them using the "UTF8" APPEND data extension. If the server also advertises the CATENATE capability (as specified in [RFC4469]), the client can use the same data extension to include such a message in a CATENATE message part. The ABNF for the APPEND data extension and CATENATE extension follows: utf8-literal = "UTF8" SP "(" literal8 ")" append-data =/ utf8-literal cat-part =/ utf8-literal A server that advertises "UTF8=APPEND" has to comply with the requirements of the IMAP base specification and [RFC5322] for message fetching. Mechanisms for 7-bit downgrading to help comply with the standards are discussed in Downgrading mechanism for Internationalized eMail Address (IMA) [RFC5504]. IMAP servers that do not advertise the "UTF8=APPEND" or "UTF8=ONLY" capability SHOULD reject an APPEND command that includes any 8-bit in the message headers with a "NO" response. Note that the "UTF8=ONLY" capability described in Section 7 implies the "UTF8=APPEND" capability. See additional information in that section. 5. UTF8=USER Capability If the "UTF8=USER" capability is advertised, that indicates the server accepts UTF-8 user names and passwords and applies SASLprep [RFC4013] to both arguments of the LOGIN command. The server MUST reject UTF-8 that fails to comply with the formal syntax in RFC 3629 [RFC3629] or if it encounters Unicode characters listed in Section 2.3 of SASLprep RFC 4013 [RFC4013]. 6. UTF8=ALL Capability The "UTF8=ALL" capability indicates all server mailboxes support UTF-8 headers. Specifically, SELECT and EXAMINE with the "UTF8" parameter will never fail with a [NOT-UTF-8] response code. Resnick & Newman Experimental [Page 7] RFC 5738 IMAP Support for UTF-8 March 2010 Note that the "UTF8=ONLY" capability described in Section 7 implies the "UTF8=ALL" capability. See additional information in that section. Note that the "UTF8=ALL" capability implies the "UTF8=ACCEPT" capability. 7. UTF8=ONLY Capability The "UTF8=ONLY" capability permits an IMAP server to advertise that it does not support the international mailbox name convention (modified UTF-7), and does not permit selection or examination of any mailbox unless the "UTF8" parameter is provided. As this is an incompatible change to IMAP, a clear warning is necessary. IMAP clients that find implementation of the "UTF8=ONLY" capability problematic are encouraged to at least detect the "UTF8=ONLY" capability and provide an informative error message to the end-user. When an IMAP mailbox internally uses UTF-8 header native storage, the down-conversion step is necessary to permit selection or examination of the mailbox in a backwards compatible fashion will become more difficult to support. Although it is hoped that deployed IMAP servers will not advertise "UTF8=ONLY" for some years, this capability is intended to minimize the disruption when legacy support finally goes away. The "UTF8=ONLY" capability implies the "UTF8=ACCEPT" capability, the "UTF8=ALL" capability, and the "UTF8=APPEND" capability. A server that advertises "UTF8=ONLY" need not advertise the three implicit capabilities. 8. Up-Conversion Server Requirements When an IMAP4 server uses a traditional mailbox format that includes 7-bit headers and it chooses to permit access to that mailbox with the "UTF8" parameter, it MUST support minimal up-conversion as described in this section. The server MUST support up-conversion of the following address header-fields in the message header: From, Sender, To, CC, Bcc, Resent-From, Resent-Sender, Resent-To, Resent-CC, Resent-Bcc, and Reply-To. This up-conversion MUST include address local-parts in fields downgraded according to [RFC5504], address domains encoded according to Internationalizing Domain Names in Applications (IDNA) [RFC3490], and MIME header encoding [RFC2047] of display-names and any [RFC5322] comments. Resnick & Newman Experimental [Page 8] RFC 5738 IMAP Support for UTF-8 March 2010 The following charsets MUST be supported for up-conversion of MIME header encoding [RFC2047]: UTF-8, US-ASCII, ISO-8859-1, ISO-8859-2, ISO-8859-3, ISO-8859-4, ISO-8859-5, ISO-8859-6, ISO-8859-7, ISO-8859-8, ISO-8859-9, ISO-8859-10, ISO-8859-14, and ISO-8859-15. If the server supports other charsets in IMAP SEARCH or IMAP CONVERT [RFC5259], it SHOULD also support those charsets in this conversion. Up-conversion of MIME header encoding of the following headers MUST also be implemented: Subject, Date ([RFC5322] comments only), Comments, Keywords, and Content-Description. Server implementations also SHOULD up-convert all MIME body headers [RFC2045], SHOULD up-convert or remove the deprecated (and misused) "name" parameter [RFC1341] on Content-Type, and MUST up-convert the Content-Disposition [RFC2183] "filename" parameter, except when any of these are contained within a multipart/signed MIME body part (see below). These parameters can be encoded using the standard MIME parameter encoding [RFC2231] mechanism, or via non-standard use of MIME header encoding [RFC2047] in quoted strings. The IMAP server MUST NOT perform up-conversion of headers and content of multipart/signed, as well as Original-Recipient and Return-Path. 9. Issues with UTF-8 Header Mailstore When an IMAP server uses a mailbox format that supports UTF-8 headers and it permits selection or examination of that mailbox without the "UTF8" parameter, it is the responsibility of the server to comply with the IMAP4rev1 base specification [RFC3501] and [RFC5322] with respect to all header information transmitted over the wire. Mechanisms for 7-bit downgrading to help comply with the standards are discussed in "Downgrading Mechanism for Email Address Internationalization" [RFC5504]. An IMAP server with a mailbox that supports UTF-8 headers MUST comply with the protocol requirements implicit from Section 8. However, the code necessary for such compliance need not be part of the IMAP server itself in this case. For example, the minimal required up- conversion could be performed when a message is inserted into the IMAP-accessible mailbox. 10. IANA Considerations This adds five new capabilities ("UTF8=ACCEPT", "UTF8=USER", "UTF8=APPEND", "UTF8=ALL", and "UTF8=ONLY") to the IMAP4rev1 Capabilities registry [RFC3501]. Resnick & Newman Experimental [Page 9] RFC 5738 IMAP Support for UTF-8 March 2010 This adds two new IMAP4 list selection options and one new IMAP4 list return option. 1. LIST-EXTENDED option name: UTF8 LIST-EXTENDED option type: SELECTION Implied return options(s): UTF8 LIST-EXTENDED option description: Causes the LIST response to include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter. Published specification: RFC 5738, Section 3.4.1 Security considerations: RFC 5738, Section 11 Intended usage: COMMON Person and email address to contact for further information: see the Authors' Addresses at the end of this specification Owner/Change controller: iesg@ietf.org 2. LIST-EXTENDED option name: UTF8ONLY LIST-EXTENDED option type: SELECTION Implied return options(s): UTF8 LIST-EXTENDED option description: Causes the LIST response to include mailboxes that mandate the UTF8 SELECT/EXAMINE parameter and exclude mailboxes that do not support the UTF8 SELECT/EXAMINE parameter. Published specification: RFC 5738, Section 3.4.1 Security considerations: RFC 5738, Section 11 Intended usage: COMMON Person and email address to contact for further information: see the Authors' Addresses at the end of this specification Owner/Change controller: iesg@ietf.org Resnick & Newman Experimental [Page 10] RFC 5738 IMAP Support for UTF-8 March 2010 3. LIST-EXTENDED option name: UTF8 LIST-EXTENDED option type: RETURN Implied return options(s): none LIST-EXTENDED option description: Causes the LIST response to include \NoUTF8 and \UTF8Only mailbox attributes. Published specification: RFC 5738, Section 3.4.1 Security considerations: RFC 5738, Section 11 Intended usage: COMMON Person and email address to contact for further information: see the Authors' Addresses at the end of this specification Owner/Change controller: iesg@ietf.org 11. Security Considerations The security considerations of UTF-8 [RFC3629] and SASLprep [RFC4013] apply to this specification, particularly with respect to use of UTF-8 in user names and passwords. Otherwise, this is not believed to alter the security considerations of IMAP4rev1. 12. References 12.1. Normative References [RFC1341] Borenstein, N. and N. Freed, "MIME (Multipurpose Internet Mail Extensions): Mechanisms for Specifying and Describing the Format of Internet Message Bodies", RFC 1341, June 1992. [RFC2045] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies", RFC 2045, November 1996. [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. Resnick & Newman Experimental [Page 11] RFC 5738 IMAP Support for UTF-8 March 2010 [RFC2183] Troost, R., Dorner, S., and K. Moore, "Communicating Presentation Information in Internet Messages: The Content-Disposition Header Field", RFC 2183, August 1997. [RFC2231] Freed, N. and K. Moore, "MIME Parameter Value and Encoded Word Extensions: Character Sets, Languages, and Continuations", RFC 2231, November 1997. [RFC3490] Faltstrom, P., Hoffman, P., and A. Costello, "Internationalizing Domain Names in Applications (IDNA)", RFC 3490, March 2003. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User Names and Passwords", RFC 4013, February 2005. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC4469] Resnick, P., "Internet Message Access Protocol (IMAP) CATENATE Extension", RFC 4469, April 2006. [RFC5161] Gulbrandsen, A. and A. Melnikov, "The IMAP ENABLE Extension", RFC 5161, March 2008. [RFC5198] Klensin, J. and M. Padlipsky, "Unicode Format for Network Interchange", RFC 5198, March 2008. [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access Protocol version 4 - LIST Command Extensions", RFC 5258, June 2008. [RFC5259] Melnikov, A. and P. Coates, "Internet Message Access Protocol - CONVERT Extension", RFC 5259, July 2008. [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, October 2008. Resnick & Newman Experimental [Page 12] RFC 5738 IMAP Support for UTF-8 March 2010 [RFC5335] Abel, Y., "Internationalized Email Headers", RFC 5335, September 2008. [RFC5504] Fujiwara, K. and Y. Yoneya, "Downgrading Mechanism for Email Address Internationalization", RFC 5504, March 2009. 12.2. Informative References [RFC2049] Freed, N. and N. Borenstein, "Multipurpose Internet Mail Extensions (MIME) Part Five: Conformance Criteria and Examples", RFC 2049, November 1996. [RFC2088] Myers, J., "IMAP4 non-synchronizing literals", RFC 2088, January 1997. [RFC2277] Alvestrand, H., "IETF Policy on Character Sets and Languages", BCP 18, RFC 2277, January 1998. [RFC5721] Gellens, R. and C. Newman, "POP3 Support for UTF-8", RFC 5721, February 2010. Resnick & Newman Experimental [Page 13] RFC 5738 IMAP Support for UTF-8 March 2010 Appendix A. Design Rationale This non-normative section discusses the reasons behind some of the design choices in the above specification. The basic approach of advertising the ability to access a mailbox in UTF-8 mode is intended to permit graceful upgrade, including servers that support multiple mailbox formats. In particular, it would be undesirable to force conversion of an entire server mailstore to UTF-8 headers, so being able to phase-in support for new mailboxes and gradually migrate old mailboxes is permitted by this design. "UTF8=USER" is optional because many identity systems are US-ASCII only, so it's helpful to inform the client up front that UTF-8 won't work. "UTF8=APPEND" is optional because it effectively requires IMAP server support for down-conversion, which is a much more complex operation than up-conversion. The "UTF8=ONLY" mechanism simplifies diagnosis of interoperability problems when legacy support goes away. In the situation where backwards compatibility is broken anyway, just-send-UTF-8 IMAP has the advantage that it might work with some legacy clients. However, the difficulty of diagnosing interoperability problems caused by a just-send-UTF-8 IMAP mechanism is the reason the "UTF8=ONLY" capability mechanism was chosen. The up-conversion requirements are designed to balance the desire to deprecate and eventually eliminate complicated encodings (like MIME header encodings) without creating a significant deployment burden for servers. As IMAP4 servers already require a MIME parser, this includes additional server up-conversion requirements not present in POP3 Support for UTF-8 [RFC5721]. The set of mandatory charsets comes from two sources: MIME requirements [RFC2049] and IETF Policy on Character Sets [RFC2277]. Including a requirement to up-convert widely deployed encoded ideographic charsets to UTF-8 would be reasonable for most scenarios, but may require unacceptable table sizes for some embedded devices. The open-ended recommendation to support widely deployed charsets avoids the political ramifications of attempting to list such charsets. The authors believe market forces, existing open-source software, and public conversion tables are sufficient to deploy the appropriate charsets. Resnick & Newman Experimental [Page 14] RFC 5738 IMAP Support for UTF-8 March 2010 Appendix B. Examples Demonstrating Relationships between UTF8= Capabilities UTF8=ACCEPT UTF8=USER UTF8=APPEND UTF8=ACCEPT UTF8=ALL UTF8=ALL ; Note, same as above UTF8=ACCEPT UTF8=USER UTF8=APPEND UTF8=ALL UTF8=ONLY UTF8=USER UTF8=ONLY ; Note, same as above Appendix C. Acknowledgments The authors wish to thank the participants of the EAI working group for their contributions to this document with particular thanks to Harald Alvestrand, David Black, Randall Gellens, Arnt Gulbrandsen, Kari Hurtta, John Klensin, Xiaodong Lee, Charles Lindsey, Alexey Melnikov, Subramanian Moonesamy, Shawn Steele, Daniel Taharlev, and Joseph Yee for their specific contributions to the discussion. Authors' Addresses Pete Resnick Qualcomm Incorporated 5775 Morehouse Drive San Diego, CA 92121-1714 US Phone: +1 858 651 4478 EMail: presnick@qualcomm.com URI: http://www.qualcomm.com/~presnick/ Chris Newman Sun Microsystems 800 Royal Oaks Monrovia, CA 91016 US EMail: chris.newman@sun.com Resnick & Newman Experimental [Page 15] ================================================ FILE: docs/rfcs/rfc5788.IMAP4_Keyword_registry.txt ================================================ Internet Engineering Task Force (IETF) A. Melnikov Request for Comments: 5788 D. Cridland Category: Standards Track Isode Limited ISSN: 2070-1721 March 2010 IMAP4 Keyword Registry Abstract The aim of this document is to establish a new IANA registry for IMAP keywords and to define a procedure for keyword registration, in order to improve interoperability between different IMAP clients. Status of This Memo This is an Internet Standards Track document. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Further information on Internet Standards is available in Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc5788. Copyright Notice Copyright (c) 2010 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Melnikov & Cridland Standards Track [Page 1] RFC 5788 IMAP4 Keyword Registry March 2010 Table of Contents 1. Introduction ....................................................2 2. Conventions Used in This Document ...............................2 3. IANA Considerations .............................................3 3.1. Review Guidelines for the Designated Expert Reviewer .......4 3.2. Comments on IMAP Keywords' Registrations ...................5 3.3. Change Control .............................................5 3.4. Initial Registrations ......................................6 3.4.1. $MDNSent IMAP Keyword Registration ..................6 3.4.2. $Forwarded IMAP Keyword Registration ................7 3.4.3. $SubmitPending IMAP Keyword Registration ............8 3.4.4. $Submitted IMAP Keyword Registration ................9 4. Security Considerations ........................................10 5. Acknowledgements ...............................................10 6. References .....................................................10 6.1. Normative References ......................................10 6.2. Informative References ....................................11 1. Introduction IMAP keywords [RFC3501] are boolean named flags that can be used by clients to annotate messages in an IMAP mailbox. Although IMAP keywords are an optional feature of IMAP, the majority of IMAP servers can store arbitrary keywords. Many mainstream IMAP clients use a limited set of specific keywords, and some can manage (create, edit, display) arbitrary IMAP keywords. Over the years, some IMAP keywords have become de-facto standards, with some specific semantics associated with them. In some cases, different client implementors decided to define and use keywords with different names, but the same semantics. Some server implementors decided to map such keywords automatically in order to improve cross- client interoperability. In other cases, the same keywords have been used with different semantics, thus causing interoperability problems. This document attempts to prevent further incompatible uses of IMAP keywords by establishing an "IMAP Keywords" registry and allocating a special prefix for standardized keywords. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [Kwds]. Melnikov & Cridland Standards Track [Page 2] RFC 5788 IMAP4 Keyword Registry March 2010 3. IANA Considerations IANA has established a new registry for IMAP keywords. Registration of an IMAP keyword is requested by filling in the following template and following the instructions on the IANA pages for the registry to submit it to IANA: Subject: Registration of IMAP keyword X IMAP keyword name: Purpose (description): Private or Shared on a server: (One of PRIVATE, SHARED, or BOTH. PRIVATE means that each different user has a distinct copy of the keyword. SHARED means that all different users see the same value of the keyword. BOTH means that an IMAP server can have the keyword as either private or shared.) Is it an advisory keyword or may it cause an automatic action: When/by whom the keyword is set/cleared: Related keywords: (for example, "mutually exclusive with keywords Y and Z") Related IMAP capabilities: Security considerations: Published specification (recommended): Person & email address to contact for further information: Intended usage: (One of COMMON, LIMITED USE, or DEPRECATED (i.e., not recommended for use)) Owner/Change controller: (MUST be "IESG" for any "common use" keyword registration specified in an IETF Review document. See definition of "common use" below in this section. When the Owner/Change controller is not a Standardization Organization, the registration request MUST make it clear if Melnikov & Cridland Standards Track [Page 3] RFC 5788 IMAP4 Keyword Registry March 2010 the registration is controlled by a company, or the individual performing the registration.) Note: (Any other information that the author deems interesting may be added here, for example, if the keyword(s) is supported by existing clients.) Registration of an IMAP keyword requires Expert Review [RFC5226]. Registration of any IMAP keyword is initiated by posting a registration request to the Message Organization WG mailing list (or its replacement as chosen by the responsible Application Area Director) and CCing IANA (). After allowing for at least two weeks for community input on the designated mailing list, the expert will determine the appropriateness of the registration request and either approve or disapprove the request with notice to the requestor, the mailing list, and IANA. Any refusal must come with a clear explanation. The IESG appoints one or more Expert Reviewers for the IMAP keyword registry established by this document. The Expert Reviewer should strive for timely reviews. The Expert Reviewer should take no longer than six weeks to make and announce the decision, or notify the mailing list that more time is required. Decisions (or lack of) made by an Expert Reviewer can be first appealed to Application Area Directors and, if the appellant is not satisfied with the response, to the full IESG. There are two types of IMAP keywords in the "IMAP Keywords" registry: intended for "common use" and vendor-/organization-specific use (also known as "limited use"). An IMAP keyword is said to be for "common use" if it is reasonably expected to be implemented in at least two independent client implementations. The two types of IMAP keywords have different levels of requirements for registration (see below). 3.1. Review Guidelines for the Designated Expert Reviewer Expert Reviewers should focus on the following requirements. Registration of a vendor-/organization-specific ("limited use") IMAP keyword is easier. The Expert Reviewer only needs to check that the requested name doesn't conflict with an already registered name, and that the name is not too generic, misleading, etc. The Expert Reviewer MAY request the IMAP keyword name change before approving Melnikov & Cridland Standards Track [Page 4] RFC 5788 IMAP4 Keyword Registry March 2010 the registration. The Expert Reviewer SHOULD refuse a registration if there is an already registered IMAP keyword that serves the same purpose, but has a different name. When registering an IMAP keyword for "common use", the Expert Reviewer performs the checks described for vendor-/ organization-specific IMAP keywords, plus additional checks as detailed below. Keywords intended for "common use" SHOULD start with the "$" prefix. (Note that this is a SHOULD because some of the commonly used IMAP keywords in widespread use don't follow this convention.) IMAP keywords intended for "common use" SHOULD be standardized in IETF Review [RFC5226] documents. (Note that IETF Review documents still require Expert Review.) Values in the "IMAP Keywords" IANA registry intended for "common use" must be clearly documented and likely to ensure interoperability. They must be useful, not harmful to the Internet. In cases when an IMAP keyword being registered is already deployed, Expert Reviewers should favor registering it over requiring perfect documentation and/or requesting a change to the name of the IMAP keyword. The Expert Reviewer MAY automatically "upgrade" registration requests for a "limited use" IMAP keyword to "common use" level. The Expert Reviewer MAY also request that a registration targeted for "common use" be registered as "limited use" instead. 3.2. Comments on IMAP Keywords' Registrations Comments on registered IMAP keywords should be sent to both the "owner" of the mechanism and to the mailing list designated to IMAP keyword review (see Section 3). This improves the chances of getting a timely response. Submitters of comments may, after a reasonable attempt to contact the owner and after soliciting comments on the IMAP mailing list, request the designated Expert Reviewer to attach their comment to the IMAP keyword registration itself. The procedure is similar to requesting an Expert Review for the affected keyword. 3.3. Change Control Once an IMAP keyword registration has been published by IANA, the owner may request a change to its definition. The change request (including a change to the "intended usage" field) follows the same procedure as the initial registration request, with the exception of Melnikov & Cridland Standards Track [Page 5] RFC 5788 IMAP4 Keyword Registry March 2010 changes to the "Person & email address to contact for further information" and "Owner/Change controller" fields. The latter can be changed by the owner by informing IANA; this can be done without discussion or review. The IESG may reassign responsibility for an IMAP keyword. The most common case of this will be to enable clarifications to be made to keywords where the owner of the registration has died, moved out of contact, or is otherwise unable to make changes that are important to the community. IMAP keyword registrations MUST NOT be deleted; keywords that are no longer believed appropriate for use can be declared DEPRECATED by a change to their "intended usage" field. The IESG is considered the owner of all "common use" IMAP keywords that are published in an IETF Review document. 3.4. Initial Registrations IANA has registered the IMAP keywords specified in following subsections in the registry established by this document. 3.4.1. $MDNSent IMAP Keyword Registration Subject: Registration of IMAP keyword $MDNSent IMAP keyword name: $MDNSent Purpose (description): Specifies that a Message Disposition Notification (MDN) must not be sent for any message annotated with the $MDNSent IMAP keyword. Private or Shared on a server: SHARED Is it an advisory keyword or may it cause an automatic action: This keyword can cause automatic action by the client. See [RFC3503] for more details. When/by whom the keyword is set/cleared: This keyword is set by an IMAP client when it decides to act on an MDN request, or when uploading a sent or draft message. It can also be set by a delivery agent. Once set, the flag SHOULD NOT be cleared. Melnikov & Cridland Standards Track [Page 6] RFC 5788 IMAP4 Keyword Registry March 2010 Related keywords: None Related IMAP capabilities: None Security considerations: See Section 6 of [RFC3503] Published specification (recommended): [RFC3503] Person & email address to contact for further information: Alexey Melnikov Intended usage: COMMON Owner/Change controller: IESG Note: 3.4.2. $Forwarded IMAP Keyword Registration Subject: Registration of the IMAP keyword $Forwarded IMAP keyword name: $Forwarded Purpose (description): $Forwarded is used by several IMAP clients to specify that the message was resent to another email address, embedded within or attached to a new message. This keyword is set by the mail client when it successfully forwards the message to another email address. Typical usage of this keyword is to show a different (or additional) icon for a message that has been forwarded. Private or Shared on a server: BOTH Is it an advisory keyword or may it cause an automatic action: advisory When/by whom the keyword is set/cleared: This keyword can be set by either a delivery agent or a mail client. Once set, the flag SHOULD NOT be cleared. Notes: There is no way to tell if a message with $Forwarded keyword set was forwarded more than once. Related keywords: None Related IMAP capabilities: None Melnikov & Cridland Standards Track [Page 7] RFC 5788 IMAP4 Keyword Registry March 2010 Security considerations: A server implementing this keyword as a shared keyword may disclose that a confidential message was forwarded. Published specification (recommended): [RFC5550] Person & email address to contact for further information: Alexey Melnikov Intended usage: COMMON Owner/Change controller: IESG Note: 3.4.3. $SubmitPending IMAP Keyword Registration Subject: Registration of IMAP keyword $SubmitPending IMAP keyword name: $SubmitPending Purpose (description): The $SubmitPending IMAP keyword designates the message as awaiting to be submitted. This keyword allows storing messages waiting to be submitted in the same mailbox where messages that were already submitted and/or are being edited are stored. A message that has both $Submitted and $SubmitPending IMAP keywords set is a message being actively submitted. Private or Shared on a server: SHARED Is it an advisory keyword or may it cause an automatic action: This keyword can cause automatic action by the client. See Section 5.10 of [RFC5550] for more details. When/by whom the keyword is set/cleared: This keyword is set by a mail client when it decides that the message needs to be sent out. Related keywords: $Submitted Related IMAP capabilities: None Melnikov & Cridland Standards Track [Page 8] RFC 5788 IMAP4 Keyword Registry March 2010 Security considerations: A server implementing this keyword as a shared keyword may disclose that a confidential message is scheduled to be sent out or is being actively sent out. Published specification (recommended): [RFC5550] Person & email address to contact for further information: Alexey Melnikov Intended usage: COMMON Owner/Change controller: IESG Note: 3.4.4. $Submitted IMAP Keyword Registration Subject: Registration of IMAP keyword $Submitted IMAP keyword name: $Submitted Purpose (description): The $Submitted IMAP keyword designates the message as being sent out. A message that has both $Submitted and $SubmitPending IMAP keywords set is a message being actively submitted. Private or Shared on a server: SHARED Is it an advisory keyword or may it cause an automatic action: This keyword can cause automatic action by the client. See Section 5.10 of [RFC5550] for more details. When/by whom the keyword is set/cleared: This keyword is set by a mail client when it decides to start sending it. Related keywords: $SubmitPending Related IMAP capabilities: None Security considerations: A server implementing this keyword as a shared keyword may disclose that a confidential message was sent or is being actively sent out. Melnikov & Cridland Standards Track [Page 9] RFC 5788 IMAP4 Keyword Registry March 2010 Published specification (recommended): [RFC5550] Person & email address to contact for further information: Alexey Melnikov Intended usage: COMMON Owner/Change controller: IESG Note: 4. Security Considerations IMAP keywords are one of the base IMAP features [RFC3501]. This document doesn't change their behavior, so it does not add new security issues. A particular IMAP keyword might have specific security considerations, which are documented in the IMAP keyword registration template standardized by this document. 5. Acknowledgements The creation of this document was prompted by one of many discussions on the IMAP mailing list. John Neystadt co-authored the first version of this document. Special thanks to Chris Newman, David Harris, Lyndon Nerenberg, Mark Crispin, Samuel Weiler, Alfred Hoenes, Lars Eggert, and Cullen Jennings for reviewing different versions of this document. However, all errors or omissions must be attributed to the authors of this document. The authors would also like to thank the developers of Mozilla mail clients for providing food for thought. 6. References 6.1. Normative References [Kwds] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. Melnikov & Cridland Standards Track [Page 10] RFC 5788 IMAP4 Keyword Registry March 2010 [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an IANA Considerations Section in RFCs", BCP 26, RFC 5226, May 2008. 6.2. Informative References [RFC3503] Melnikov, A., "Message Disposition Notification (MDN) profile for Internet Message Access Protocol (IMAP)", RFC 3503, March 2003. [RFC5550] Cridland, D., Melnikov, A., and S. Maes, "The Internet Email to Support Diverse Service Environments (Lemonade) Profile", RFC 5550, August 2009. Authors' Addresses Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Dave Cridland Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: dave.cridland@isode.com Melnikov & Cridland Standards Track [Page 11] ================================================ FILE: docs/rfcs/rfc5819.IMAP4_extension_Returning_STATUS_info_in_LIST.txt ================================================ Internet Engineering Task Force (IETF) A. Melnikov Request for Comments: 5819 Isode Limited Category: Standards Track T. Sirainen ISSN: 2070-1721 Unaffiliated March 2010 IMAP4 Extension for Returning STATUS Information in Extended LIST Abstract Many IMAP clients display information about total number of messages / total number of unseen messages in IMAP mailboxes. In order to do that, they are forced to issue a LIST or LSUB command and to list all available mailboxes, followed by a STATUS command for each mailbox found. This document provides an extension to LIST command that allows the client to request STATUS information for mailboxes together with other information typically returned by the LIST command. Status of This Memo This is an Internet Standards Track document. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Further information on Internet Standards is available in Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc5819. Copyright Notice Copyright (c) 2010 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Melnikov & Sirainen Standards Track [Page 1] RFC 5819 TITLE* March 2010 Table of Contents 1. Introduction ....................................................2 1.1. Conventions Used in This Document ..........................2 2. STATUS Return Option to LIST Command ............................2 3. Examples ........................................................3 4. Formal Syntax ...................................................4 5. Security Considerations .........................................4 6. IANA Considerations .............................................4 7. Acknowledgements ................................................5 8. Normative References ............................................5 1. Introduction Many IMAP clients display information about the total number of messages / total number of unseen messages in IMAP mailboxes. In order to do that, they are forced to issue a LIST or LSUB command and to list all available mailboxes, followed by a STATUS command for each mailbox found. This document provides an extension to LIST command that allows the client to request STATUS information for mailboxes together with other information typically returned by the LIST command. 1.1. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [Kwds]. 2. STATUS Return Option to LIST Command [RFC3501] explicitly disallows mailbox patterns in the STATUS command. The main reason was to discourage frequent use of the STATUS command by clients, as it might be quite expensive for an IMAP server to perform. However, this prohibition had resulted in an opposite effect: a new generation of IMAP clients appeared, that issues a STATUS command for each mailbox returned by the LIST command. This behavior is suboptimal to say at least. It wastes extra bandwidth and, in the case of a client that doesn't support IMAP pipelining, also degrades performance by using too many round trips. This document tries to remedy the situation by specifying a single command that can be used by the client to request all the necessary information. In order to achieve this goal, this document is extending the LIST command with a new return option, STATUS. This option takes STATUS data items as parameters. For each selectable Melnikov & Sirainen Standards Track [Page 2] RFC 5819 TITLE* March 2010 mailbox matching the list pattern and selection options, the server MUST return an untagged LIST response followed by an untagged STATUS response containing the information requested in the STATUS return option. If an attempted STATUS for a listed mailbox fails because the mailbox can't be selected (e.g., if the "l" ACL right [ACL] is granted to the mailbox and the "r" right is not granted, or due to a race condition between LIST and STATUS changing the mailbox to \NoSelect), the STATUS response MUST NOT be returned and the LIST response MUST include the \NoSelect attribute. This means the server may have to buffer the LIST reply until it has successfully looked up the necessary STATUS information. If the server runs into unexpected problems while trying to look up the STATUS information, it MAY drop the corresponding STATUS reply. In such a situation, the LIST command would still return a tagged OK reply. 3. Examples C: A01 LIST "" % RETURN (STATUS (MESSAGES UNSEEN)) S: * LIST () "." "INBOX" S: * STATUS "INBOX" (MESSAGES 17 UNSEEN 16) S: * LIST () "." "foo" S: * STATUS "foo" (MESSAGES 30 UNSEEN 29) S: * LIST (\NoSelect) "." "bar" S: A01 OK List completed. The "bar" mailbox isn't selectable, so it has no STATUS reply. C: A02 LIST (SUBSCRIBED RECURSIVEMATCH)"" % RETURN (STATUS (MESSAGES)) S: * LIST (\Subscribed) "." "INBOX" S: * STATUS "INBOX" (MESSAGES 17) S: * LIST () "." "foo" (CHILDINFO ("SUBSCRIBED")) S: A02 OK List completed. The LIST reply for "foo" is returned because it has matching children, but no STATUS reply is returned because "foo" itself doesn't match the selection criteria. Melnikov & Sirainen Standards Track [Page 3] RFC 5819 TITLE* March 2010 4. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. Terms not defined here are taken from [RFC3501] and [LISTEXT]. return-option =/ status-option status-option = "STATUS" SP "(" status-att *(SP status-att) ")" ;; This ABNF production complies with ;; syntax. 5. Security Considerations This extension makes it a bit easier for clients to overload the server by requesting STATUS information for a large number of mailboxes. However, as already noted in the introduction, existing clients already try to do that by generating a large number of STATUS commands for each mailbox in which they are interested. While performing STATUS information retrieval for big lists of mailboxes, a server implementation needs to make sure that it can still serve other IMAP connections and yield execution to other connections, when necessary. 6. IANA Considerations IMAP4 capabilities are registered by publishing a Standards Track or IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry is available from the IANA webiste: http://www.iana.org This document defines the LIST-STATUS IMAP capability. IANA has added it to the registry. IANA has also added the following new LIST-EXTENDED option to the IANA registry established by [LISTEXT]: To: iana@iana.org Subject: Registration of LIST-EXTENDED option STATUS LIST-EXTENDED option name: STATUS LIST-EXTENDED option type: RETURN LIST-EXTENDED option description: Causes the LIST command to return STATUS responses in addition to LIST responses. Melnikov & Sirainen Standards Track [Page 4] RFC 5819 TITLE* March 2010 Published specification: RFC 5819 Security considerations: RFC 5819 Intended usage: COMMON Person and email address to contact for further information: Alexey Melnikov Owner/Change controller: iesg@ietf.org 7. Acknowledgements Thanks to Philip Van Hoof who pointed out that STATUS and LIST commands should be combined in order to optimize traffic and number of round trips. 8. Normative References [ABNF] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [ACL] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [Kwds] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [LISTEXT] Leiba, B. and A. Melnikov, "Internet Message Access Protocol version 4 - LIST Command Extensions", RFC 5258, June 2008. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. Melnikov & Sirainen Standards Track [Page 5] RFC 5819 TITLE* March 2010 Authors' Addresses Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Timo Sirainen EMail: tss@iki.fi Melnikov & Sirainen Standards Track [Page 6] ================================================ FILE: docs/rfcs/rfc5957.IMAP4_SORT_extension.txt ================================================ Internet Engineering Task Force (IETF) D. Karp Request for Comments: 5957 Zimbra Updates: 5256 July 2010 Category: Standards Track ISSN: 2070-1721 Display-Based Address Sorting for the IMAP4 SORT Extension Abstract This document describes an IMAP protocol extension enabling server- side message sorting on the commonly displayed portion of the From and To header fields. Status of This Memo This is an Internet Standards Track document. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Further information on Internet Standards is available in Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc5957. Copyright Notice Copyright (c) 2010 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Karp IMAP4 Display-Based Address Sorting [Page 1] RFC 5957 July 2010 Table of Contents 1. Introduction ....................................................2 2. Conventions Used in This Document ...............................2 3. DISPLAY Sort Value for an Address ...............................2 4. The DISPLAYFROM and DISPLAYTO Sort Criteria .....................3 5. Formal Syntax ...................................................3 6. Security Considerations .........................................3 7. Internationalization Considerations .............................4 8. IANA Considerations .............................................4 9. Normative References ............................................4 1. Introduction The [SORT] extension to the [IMAP] protocol provides a means for the server-based sorting of messages. It defines a set of sort criteria and the mechanism for determining the sort value of a message for each such ordering. The [SORT] FROM and TO orderings sort messages lexically on the [IMAP] addr-mailbox of the first address in the message's From and To headers, respectively. This document provides two alternative orderings, DISPLAYFROM and DISPLAYTO, which sort messages based on the first From or To address's [IMAP] addr-name (generally the same as its [RFC5322] display-name), when present. A server that supports the full [SORT] extension as well as both the DISPLAYFROM and DISPLAYTO sort criteria indicates this by returning "SORT=DISPLAY" in its CAPABILITY response. 2. Conventions Used in This Document The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC2119]. 3. DISPLAY Sort Value for an Address For the purposes of the sort criteria defined in this document, the sort value for an [IMAP] address structure is defined as follows: o If the address structure's [IMAP] addr-name is non-NIL, apply the procedure from [RFC5255], Section 4.6. (That is, decode any [RFC2047] encoded-words and convert the resulting character string into a charset valid for the currently active [RFC4790] collation, with a default of UTF-8.) If the resulting octet string is not the empty string, use it as the sort value for the address. Karp IMAP4 Display-Based Address Sorting [Page 2] RFC 5957 July 2010 o Otherwise, if the address structure's [IMAP] addr-mailbox and [IMAP] addr-host are both non-NIL, the sort value for the address is addr-mailbox@addr-host. o Otherwise, if the address structure's [IMAP] addr-mailbox is non- NIL, the sort value for the address is its addr-mailbox. o If none of the above conditions are met, the sort value for the address is the empty string. 4. The DISPLAYFROM and DISPLAYTO Sort Criteria This document introduces two new [SORT] sort criteria, DISPLAYFROM and DISPLAYTO. A message's sort value under these orderings MUST be derived as follows: A "derived-addr" value is created from the [IMAP] envelope structure resulting from a FETCH ENVELOPE on the message. For DISPLAYFROM, the derived-addr value is the [IMAP] env-from value. For DISPLAYTO, the derived-addr value is the [IMAP] env-to value. o If the derived-addr value is NIL, the message's sort value is the empty string. o Otherwise, the message's sort value is the DISPLAY sort value of the first [IMAP] address in the derived-addr value. 5. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) notation as specified in [RFC5234]. [IMAP] defines the non-terminal "capability", and [SORT] defines "sort-key". capability =/ "SORT=DISPLAY" sort-key =/ "DISPLAYFROM" / "DISPLAYTO" 6. Security Considerations This document defines an additional IMAP4 capability. As such, it does not change the underlying security considerations of [IMAP]. The author believes that no new security issues are introduced with this additional IMAP4 capability. Karp IMAP4 Display-Based Address Sorting [Page 3] RFC 5957 July 2010 7. Internationalization Considerations DISPLAYFROM and DISPLAYTO are string-based sort criteria. As stated in [SORT], the active [RFC4790] collation as per [RFC5255] MUST be used when sorting such strings. The DISPLAYFROM and DISPLAYTO orderings sort on the full decoded [IMAP] addr-name, when present. They do not attempt to parse this string in a locale- or language-dependent manner in order to determine and sort on some semantically meaningful substring such as the surname. 8. IANA Considerations [IMAP] capabilities are registered by publishing a Standards Track or IESG-approved Experimental RFC. This document constitutes registration of the SORT=DISPLAY capability in the [IMAP] capabilities registry. 9. Normative References [IMAP] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC2047] Moore, K., "MIME (Multipurpose Internet Mail Extensions) Part Three: Message Header Extensions for Non-ASCII Text", RFC 2047, November 1996. [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC4790] Newman, C., Duerst, M., and A. Gulbrandsen, "Internet Application Protocol Collation Registry", RFC 4790, March 2007. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet Message Access Protocol Internationalization", RFC 5255, June 2008. [RFC5322] Resnick, P., Ed., "Internet Message Format", RFC 5322, October 2008. Karp IMAP4 Display-Based Address Sorting [Page 4] RFC 5957 July 2010 [SORT] Crispin, M. and K. Murchison, "Internet Message Access Protocol - SORT and THREAD Extensions", RFC 5256, June 2008. Author's Address Dan Karp Zimbra 3401 Hillview Avenue Palo Alto, CA 94304 USA EMail: dkarp@zimbra.com URI: http://www.zimbra.com Karp IMAP4 Display-Based Address Sorting [Page 5] ================================================ FILE: docs/rfcs/rfc6154.IMAP_LIST_Special-use_Mailboxes.txt ================================================ Internet Engineering Task Force (IETF) B. Leiba Request for Comments: 6154 Huawei Technologies Category: Standards Track J. Nicolson ISSN: 2070-1721 Google March 2011 IMAP LIST Extension for Special-Use Mailboxes Abstract Some IMAP message stores include special-use mailboxes, such as those used to hold draft messages or sent messages. Many mail clients allow users to specify where draft or sent messages should be put, but configuring them requires that the user know which mailboxes the server has set aside for these purposes. This extension adds new optional mailbox attributes that a server may include in IMAP LIST command responses to identify special-use mailboxes to the client, easing configuration. Status of This Memo This is an Internet Standards Track document. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Further information on Internet Standards is available in Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc6154. Copyright Notice Copyright (c) 2011 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Leiba & Nicolson Standards Track [Page 1] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 1.1. Conventions Used in This Document . . . . . . . . . . . . 3 2. New Mailbox Attributes Identifying Special-Use Mailboxes . . . 3 3. Extension to IMAP CREATE Command to Set Special-Use Attributes . . . . . . . . . . . . . . . . . . . . . . . . . . 5 4. IMAP METADATA Entry for Special-Use Attributes . . . . . . . . 6 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 7 5.1. Example of an IMAP LIST Command . . . . . . . . . . . . . 7 5.2. Example of an Extended IMAP LIST Command . . . . . . . . . 7 5.3. Example of an IMAP CREATE Command . . . . . . . . . . . . 8 5.4. Example of Using IMAP METADATA to Manipulate Special-Use Attributes . . . . . . . . . . . . . . . . . . 8 6. Formal Syntax . . . . . . . . . . . . . . . . . . . . . . . . 9 7. Security Considerations . . . . . . . . . . . . . . . . . . . 9 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 10 8.1. Registration of USEATTR IMAP Response Code . . . . . . . . 10 8.2. Registration of CREATE-SPECIAL-USE IMAP Capability . . . . 10 8.3. Registration of SPECIAL-USE IMAP Capability . . . . . . . 10 8.4. Registration of SPECIAL-USE Selection Option . . . . . . . 10 8.5. Registration of SPECIAL-USE Return Option . . . . . . . . 11 8.6. Registration of SPECIAL-USE Metadata . . . . . . . . . . . 11 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 12 9.1. Normative References . . . . . . . . . . . . . . . . . . . 12 9.2. Informative References . . . . . . . . . . . . . . . . . . 12 Leiba & Nicolson Standards Track [Page 2] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 1. Introduction Some IMAP message stores include special-use mailboxes, such as those used to hold draft messages or sent messages. Many mail clients allow users to specify where draft or sent messages should be put, but configuring them requires that the user know which mailboxes the server has set aside for these purposes. This extension adds new optional mailbox attributes that a server may include in IMAP LIST command responses to identify special-use mailboxes to the client, easing configuration. In addition, this extension adds an optional parameter on the IMAP CREATE command, allowing a client to assign a special use to a mailbox when it is created. Servers may choose to support this part of the extension, but are not required to. 1.1. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [RFC2119]. 2. New Mailbox Attributes Identifying Special-Use Mailboxes An IMAP server that supports this extension MAY include any or all of the following attributes in responses to the non-extended IMAP LIST command. The new attributes are included along with existing attributes, such as "\Marked" and "\Noselect". A given mailbox may have none, one, or more than one of these attributes. In some cases, a special use is advice to a client about what to put in that mailbox. In other cases, it's advice to a client about what to expect to find there. There is no capability string related to the support of special-use attributes on the non-extended LIST command. For the extended list command [RFC5258], this extension adds a new capability string, a new selection option, and a new return option, all called "SPECIAL-USE". Supporting implementations MUST include the "SPECIAL-USE" capability string in response to an IMAP CAPABILITY command. If the client specifies the "SPECIAL-USE" selection option, the LIST command MUST return only those mailboxes that have a special-use attribute set. If the client specifies the "SPECIAL-USE" return option, the LIST command MUST return the new special-use attributes on those mailboxes that have them set. The "SPECIAL-USE" Leiba & Nicolson Standards Track [Page 3] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 return option is implied by the "SPECIAL-USE" selection option. The extended LIST command MAY return SPECIAL-USE attributes even if the client does not specify the return option. The new attributes defined here are as follows: \All This mailbox presents all messages in the user's message store. Implementations MAY omit some messages, such as, perhaps, those in \Trash and \Junk. When this special use is supported, it is almost certain to represent a virtual mailbox. \Archive This mailbox is used to archive messages. The meaning of an "archival" mailbox is server-dependent; typically, it will be used to get messages out of the inbox, or otherwise keep them out of the user's way, while still making them accessible. \Drafts This mailbox is used to hold draft messages -- typically, messages that are being composed but have not yet been sent. In some server implementations, this might be a virtual mailbox, containing messages from other mailboxes that are marked with the "\Draft" message flag. Alternatively, this might just be advice that a client put drafts here. \Flagged This mailbox presents all messages marked in some way as "important". When this special use is supported, it is likely to represent a virtual mailbox collecting messages (from other mailboxes) that are marked with the "\Flagged" message flag. \Junk This mailbox is where messages deemed to be junk mail are held. Some server implementations might put messages here automatically. Alternatively, this might just be advice to a client-side spam filter. \Sent This mailbox is used to hold copies of messages that have been sent. Some server implementations might put messages here automatically. Alternatively, this might just be advice that a client save sent messages here. \Trash This mailbox is used to hold messages that have been deleted or marked for deletion. In some server implementations, this might be a virtual mailbox, containing messages from other mailboxes Leiba & Nicolson Standards Track [Page 4] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 that are marked with the "\Deleted" message flag. Alternatively, this might just be advice that a client that chooses not to use the IMAP "\Deleted" model should use this as its trash location. In server implementations that strictly expect the IMAP "\Deleted" model, this special use is likely not to be supported. All of the above attributes are OPTIONAL, and any given server or message store may support any combination of the attributes, or none at all. In most cases, there will likely be at most one mailbox with a given attribute for a given user, but in some server or message store implementations it might be possible for multiple mailboxes to have the same special-use attribute. Special-use attributes are likely to be user-specific. User Adam might share his \Sent mailbox with user Barb, but that mailbox is unlikely to also serve as Barb's \Sent mailbox. It's certainly possible for Adam and Barb to each set the \Sent use on the same mailbox, but that would be done by specific action (see the sections below). 3. Extension to IMAP CREATE Command to Set Special-Use Attributes As an OPTIONAL feature, a server MAY allow clients to designate a mailbox, at creation, as having one or more special uses. This extension defines the "USE" parameter to the IMAP CREATE command for that purpose (using the syntax defined in RFC 4466 section 2.2 [RFC4466]). The new OPTIONAL "USE" parameter is followed by a parenthesized list of zero or more special-use attributes, as defined above. In some server implementations, some special uses may imply automatic action by the server. For example, creation of a "\Junk" mailbox might cause the server to start placing messages that have been evaluated as spam into the mailbox. In some server implementations, some special uses may result in a mailbox with unusual characteristics or side effects. For example, creation of an "\All" mailbox might cause the server to create a virtual mailbox, rather than a standard one, and that mailbox might behave in unexpected ways (COPY into it might fail, for example). Servers MAY allow the creation of a special-use mailbox even if one so designated already exists. This might have the effect of moving the special use from the old mailbox to the new one, or might create multiple mailboxes with the same special use. Alternatively, servers MAY refuse the creation, considering the designation to be a conflict. Leiba & Nicolson Standards Track [Page 5] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 If the server cannot create a mailbox with the designated special use defined, for whatever reason, it MUST NOT create the mailbox, and MUST respond to the CREATE command with a tagged NO response. If the reason for the failure is related to the special-use attribute (the specified special use is not supported or cannot be assigned to the specified mailbox), the server SHOULD include the new "USEATTR" response code in the tagged response (see Section 5.3 for an example). An IMAP server that supports this OPTIONAL feature will advertise the "CREATE-SPECIAL-USE" capability string. Clients MUST NOT use the "USE" parameter unless the server advertises the capability. Note that this capability string is different from the "SPECIAL-USE" string defined above, and a server that supports both functions MUST advertise both capability strings. 4. IMAP METADATA Entry for Special-Use Attributes If a server supports this extension and the METADATA extension [RFC5464], it SHOULD tie the special-use attributes for a mailbox to its metadata entry "/private/specialuse". The value of /private/ specialuse is either NIL (if there are no special-use attributes for that mailbox) or a space-separated list of special-use attributes, presented the same way they would be presented in the LIST command response. Such a server MAY allow the setting of special-use attributes through the METADATA mechanisms, thereby allowing clients to change the special uses of existing mailboxes. These changes might have side effects, as the server automatically adjusts the special uses accordingly, just as it might do with CREATE USE, above. See Section 5.4 for an example. A server that supports this MUST check the validity of changes to the special-use attributes that are done through the metadata in the same way that it checks validity for the CREATE command and for any internal mechanisms for setting special uses on mailboxes. It MUST NOT just blindly accept setting of these metadata by clients, which might result in the setting of special uses that the implementation does not support, multiple mailboxes with the same special use, or other situations that the implementation considers invalid. Leiba & Nicolson Standards Track [Page 6] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 5. Examples 5.1. Example of an IMAP LIST Command This example shows an IMAP LIST response from a server that supports this extension. Note that not all of the attributes are used. This server also supports the Child Mailbox extension [RFC3348]. C: t1 LIST "" "%" S: * LIST (\Marked \HasNoChildren) "/" Inbox S: * LIST (\HasNoChildren) "/" ToDo S: * LIST (\HasChildren) "/" Projects S: * LIST (\Sent \HasNoChildren) "/" SentMail S: * LIST (\Marked \Drafts \HasNoChildren) "/" MyDrafts S: * LIST (\Trash \HasNoChildren) "/" Trash S: t1 OK done 5.2. Example of an Extended IMAP LIST Command This example shows an IMAP LIST response from a server that supports this extension. The client uses the extended IMAP LIST command. C: t1 CAPABILITY S: * CAPABILITY IMAP4rev1 SPECIAL-USE S: t1 OK done C: t2 LIST "" "%" RETURN (SPECIAL-USE) S: * LIST (\Marked) "/" Inbox S: * LIST () "/" ToDo S: * LIST () "/" Projects S: * LIST (\Sent) "/" SentMail S: * LIST (\Marked \Drafts) "/" MyDrafts S: * LIST (\Trash) "/" Trash S: t2 OK done Here, the client also includes the "SPECIAL-USE" selection option for the same list. The "SPECIAL-USE" return option could also have been specified, but it is unnecessary, as it is implied by the selection option. Note that in this case, mailboxes that do not have a special-use attribute are not listed. Also note that we've used the wildcard "*", rather than "%", to make sure we see all special-use mailboxes, even ones that might not be at the namespace's root. C: t3 LIST (SPECIAL-USE) "" "*" S: * LIST (\Sent) "/" SentMail S: * LIST (\Marked \Drafts) "/" MyDrafts S: * LIST (\Trash) "/" Trash S: t3 OK done Leiba & Nicolson Standards Track [Page 7] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 5.3. Example of an IMAP CREATE Command This example shows an IMAP CREATE command that might be used to create a mailbox designated to hold draft and sent messages. It also attempts to create a mailbox that will contain all the user's messages, but the server does not support that special use for this user's message store. C: t1 CAPABILITY S: * CAPABILITY IMAP4rev1 CREATE-SPECIAL-USE S: t1 OK done C: t2 CREATE MySpecial (USE (\Drafts \Sent)) S: t2 OK MySpecial created C: t3 CREATE Everything (USE (\All)) S: t3 NO [USEATTR] \All not supported 5.4. Example of Using IMAP METADATA to Manipulate Special-Use Attributes This example shows how IMAP METADATA can be used to manipulate special-use attributes, if the operation is supported on the server. ==> Starting point: C: t1 LIST "" "%" RETURN (SPECIAL-USE) S: * LIST (\Sent) "/" SentMail S: * LIST (\Drafts) "/" MyDrafts S: * LIST () "/" SavedDrafts S: * LIST (\Trash) "/" Trash S: t1 OK done ==> Demonstrate the connection: C: t2 GETMETADATA "MyDrafts" /private/specialuse S: * METADATA "MyDrafts" (/private/specialuse "\\Drafts") S: t2 OK done ==> Set new use for SavedDrafts; MyDrafts changes automatically: C: t3 SETMETADATA "SavedDrafts" (/private/specialuse "\\Drafts") S: * METADATA "MyDrafts" (/private/specialuse NIL) S: t3 OK SETMETADATA complete ==> Remove special use for SentMail: C: t4 SETMETADATA "SentMail" (/private/specialuse NIL) S: t4 OK SETMETADATA complete Leiba & Nicolson Standards Track [Page 8] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 ==> Check the results: C: t5 LIST "" "%" RETURN (SPECIAL-USE) S: * LIST () "/" SentMail S: * LIST () "/" MyDrafts S: * LIST (\Drafts) "/" SavedDrafts S: * LIST (\Trash) "/" Trash S: t5 OK done 6. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [RFC5234]. create-param =/ "USE" SP "(" [use-attr *(SP use-attr)] ")" ; Extends "create-param" from RFC 4466 [RFC4466] mbx-list-oflag =/ use-attr ; Extends "mbx-list-oflag" from IMAP base [RFC3501] list-select-independent-opt =/ "SPECIAL-USE" ; Extends "list-select-independent-opt" from ; LIST-extended [RFC5258] return-option =/ "SPECIAL-USE" ; Extends "return-option" from ; LIST-extended [RFC5258] resp-text-code =/ "USEATTR" ; Extends "resp-text-code" from ; IMAP [RFC3501] use-attr = "\All" / "\Archive" / "\Drafts" / "\Flagged" / "\Junk" / "\Sent" / "\Trash" / use-attr-ext use-attr-ext = "\" atom ; Reserved for future extensions. Clients ; MUST ignore list attributes they do not understand ; Server implementations MUST NOT generate ; extension attributes except as defined by ; future Standards-Track revisions of or ; extensions to this specification. 7. Security Considerations LIST response: Conveying special-use information to a client exposes a small bit of extra information that could be of value to an attacker. Knowing, for example, that a particular mailbox (\All) contains pointers to Leiba & Nicolson Standards Track [Page 9] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 every message the user has might be of particular value. If the IMAP channel is not protected from passive eavesdropping, this could be an issue. CREATE command "USE" parameter and metadata extension: In some server implementations, some special uses may imply automatic action by the server. For example, creation of a "\Junk" mailbox might cause the server to start placing messages that have been evaluated as spam into the mailbox. Implementors SHOULD consider the consequences of allowing a user (or client program) to designate the target of such automatic action. Example: If a user is allowed to give the "\Junk" attribute to a shared mailbox, legitimate mail that's misclassified as junk (false positives) will be put into that shared mailbox, exposing the user's private mail to others. The server might warn a user of that possibility, or might refuse to allow the specification to be made on a shared mailbox. (Note that this problem exists independent of this specification, if the server allows a user to share a mailbox that's already in use for such a function.) 8. IANA Considerations 8.1. Registration of USEATTR IMAP Response Code This document defines a new IMAP response code, "USEATTR", which IANA added to the IMAP Response Codes registry. 8.2. Registration of CREATE-SPECIAL-USE IMAP Capability This document defines a new IMAP capability, "CREATE-SPECIAL-USE", which IANA added to the IMAP 4 Capabilities registry. 8.3. Registration of SPECIAL-USE IMAP Capability This document defines a new IMAP capability, "SPECIAL-USE", which IANA added to the IMAP 4 Capabilities registry. 8.4. Registration of SPECIAL-USE Selection Option This document defines a new IMAP4 List Extended selection option, "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, as follows: To: iana@iana.org Subject: Registration of LIST-EXTENDED selection option SPECIAL-USE LIST-EXTENDED option name: SPECIAL-USE LIST-EXTENDED option type: SELECTION Leiba & Nicolson Standards Track [Page 10] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 Implied return option(s): SPECIAL-USE LIST-EXTENDED option description: Limit the list to special-use mailboxes only Published specification: RFC 6154 Security considerations: none Intended usage: COMMON Person and email address to contact for further information: Authors' Addresses at the end of RFC 6154 Owner/Change controller: iesg@ietf.org 8.5. Registration of SPECIAL-USE Return Option This document defines a new IMAP4 List Extended return option, "SPECIAL-USE", which IANA added to the IMAP4 List Extended registry, as follows: To: iana@iana.org Subject: Registration of LIST-EXTENDED return option SPECIAL-USE LIST-EXTENDED option name: SPECIAL-USE LIST-EXTENDED option type: RETURN LIST-EXTENDED option description: Request special-use mailbox information Published specification: RFC 6154 Security considerations: none Intended usage: COMMON Person and email address to contact for further information: Authors' Addresses at the end of RFC 6154 Owner/Change controller: iesg@ietf.org 8.6. Registration of SPECIAL-USE Metadata This document defines a new IMAP METADATA entry. IANA added the following to the IMAP METADATA Mailbox Entry registry: To: iana@iana.org Subject: IMAP METADATA Entry Registration Type: Mailbox Name: /private/specialuse Description: Defines any special-use features of a mailbox. See the reference specification for details of its use. Content-type: text/plain; charset=us-ascii RFC Number: RFC 6154 Contact: MORG mailing list mailto:morg@ietf.org Leiba & Nicolson Standards Track [Page 11] RFC 6154 IMAP LIST: Special-Use Mailboxes March 2011 9. References 9.1. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5258] Leiba, B. and A. Melnikov, "Internet Message Access Protocol version 4 - LIST Command Extensions", RFC 5258, June 2008. [RFC5464] Daboo, C., "The IMAP METADATA Extension", RFC 5464, February 2009. 9.2. Informative References [RFC3348] Gahrns, M. and R. Cheng, "The Internet Message Action Protocol (IMAP4) Child Mailbox Extension", RFC 3348, July 2002. Authors' Addresses Barry Leiba Huawei Technologies Phone: +1 646 827 0648 EMail: barryleiba@computer.org URI: http://internetmessagingtechnology.org/ Jamie Nicolson Google EMail: nicolson@google.com Leiba & Nicolson Standards Track [Page 12] ================================================ FILE: docs/rfcs/rfc6203.IMAP4_Fuzzy_SEARCH_extension.txt ================================================ Internet Engineering Task Force (IETF) T. Sirainen Request for Comments: 6203 March 2011 Category: Standards Track ISSN: 2070-1721 IMAP4 Extension for Fuzzy Search Abstract This document describes an IMAP protocol extension enabling a server to perform searches with inexact matching and assigning relevancy scores for matched messages. Status of This Memo This is an Internet Standards Track document. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Further information on Internet Standards is available in Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc6203. Copyright Notice Copyright (c) 2011 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Sirainen Standards Track [Page 1] RFC 6203 IMAP4 FUZZY Search March 2011 1. Introduction When humans perform searches in IMAP clients, they typically want to see the most relevant search results first. IMAP servers are able to do this in the most efficient way when they're free to internally decide how searches should match messages. This document describes a new SEARCH=FUZZY extension that provides such functionality. 2. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [KEYWORDS]. 3. The FUZZY Search Key The FUZZY search key takes another search key as its argument. The server is allowed to perform all matching in an implementation- defined manner for this search key, including ignoring the active comparator as defined by [RFC5255]. Typically, this would be used to search for strings. For example: C: A1 SEARCH FUZZY (SUBJECT "IMAP break") S: * SEARCH 1 5 10 S: A1 OK Search completed. Besides matching messages with a subject of "IMAP break", the above search may also match messages with subjects "broken IMAP", "IMAP is broken", or anything else the server decides that might be a good match. This example does a fuzzy SUBJECT search, but a non-fuzzy FROM search: C: A2 SEARCH FUZZY SUBJECT work FROM user@example.com S: * SEARCH 1 4 S: A2 OK Search completed. How the server handles multiple separate FUZZY search keys is implementation-defined. Fuzzy search algorithms might change, or the results of the algorithms might be different from search to search, so that fuzzy searches with the same parameters might give different results for 1) the same user at different times, 2) different users (searches Sirainen Standards Track [Page 2] RFC 6203 IMAP4 FUZZY Search March 2011 executed simultaneously), or 3) different users (searches executed at different times). For example, a fuzzy search might adapt to a user's search habits in an attempt to give more relevant results (in a "learning" manner). Such differences can also occur because of operational decisions, such as load balancing. Clients asking for "fuzzy" really are requesting search results in a not-necessarily- deterministic way and need to give the user appropriate warning about that. 4. Relevancy Scores for Search Results Servers SHOULD assign a search relevancy score for each matched message when the FUZZY search key is given. Relevancy scores are given in the range 1-100, where 100 is the highest relevancy. The relevancy scores SHOULD use the full 1-100 range, so that clients can show them to users in a meaningful way, e.g., as a percentage value. As the name already indicates, relevancy scores specify how relevant to the search the matched message is. It's not necessarily the same as how precisely the message matched. For example, a message whose subject fuzzily matches the search string might get a higher relevancy score than a message whose body had the exact string in the middle of a sentence. When multiple search keys are matched fuzzily, how the relevancy score is calculated is server-dependent. If the server also advertises the ESEARCH capability as defined by [ESEARCH], the relevancy scores can be retrieved using the new RELEVANCY return option for SEARCH: C: B1 SEARCH RETURN (RELEVANCY ALL) FUZZY TEXT "Helo" S: * ESEARCH (TAG "B1") ALL 1,5,10 RELEVANCY (4 99 42) S: B1 OK Search completed. In the example above, the server would treat "hello", "help", and other similar strings as fuzzily matching the misspelled "Helo". The RELEVANCY return option MUST NOT be used unless a FUZZY search key is also given. Note that SEARCH results aren't sorted by relevancy; SORT is needed for that. 5. Fuzzy Matching with Non-String Search Keys Fuzzy matching is not limited to just string matching. All search keys SHOULD be matched fuzzily, although exactly what that means for different search keys is left for server implementations to decide -- including deciding that fuzzy matching is meaningless for a particular key, and falling back to exact matching. Some suggestions are given below. Sirainen Standards Track [Page 3] RFC 6203 IMAP4 FUZZY Search March 2011 Dates: A typical example could be when a user wants to find a message "from Dave about a week ago". A client could perform this search using SEARCH FUZZY (FROM "Dave" SINCE 21-Jan-2009 BEFORE 24-Jan-2009). The server could return messages outside the specified date range, but the further away the message is, the lower the relevancy score. Sizes: These should be handled similarly to dates. If a user wants to search for "about 1 MB attachments", the client could do this by sending SEARCH FUZZY (LARGER 900000 SMALLER 1100000). Again, the further away the message size is from the specified range, the lower the relevancy score. Flags: If other search criteria match, the server could return messages that don't have the specified flags set, but with lower relevancy scores. SEARCH SUBJECT "xyz" FUZZY ANSWERED, for example, might be useful if the user thinks the message he is looking for has the ANSWERED flag set, but he isn't sure. Unique Identifiers (UIDs), sequences, modification sequences: These are examples of keys for which exact matching probably makes sense. Alternatively, a server might choose, for instance, to expand a UID range by 5% on each side. 6. Extensions to SORT and SEARCH If the server also advertises the SORT capability as defined by [SORT], the results can be sorted by the new RELEVANCY sort criteria: C: C1 SORT (RELEVANCY) UTF-8 FUZZY SUBJECT "Helo" S: * SORT 5 10 1 S: C1 OK Sort completed. The message with the highest score is returned first. As with the RELEVANCY return option, RELEVANCY sort criteria MUST NOT be used unless a FUZZY search key is also given. If the server also advertises the ESORT capability as defined by [CONTEXT], the relevancy scores can be retrieved using the new RELEVANCY return option for SORT: C: C2 SORT RETURN (RELEVANCY ALL) (RELEVANCY) UTF-8 FUZZY TEXT "Helo" S: * ESEARCH (TAG "C2") ALL 5,10,1 RELEVANCY (99 42 4) S: C2 OK Sort completed. Sirainen Standards Track [Page 4] RFC 6203 IMAP4 FUZZY Search March 2011 Furthermore, if the server advertises the CONTEXT=SORT (or CONTEXT=SEARCH) capability, then the client can limit the number of returned messages to a SORT (or a SEARCH) by using the PARTIAL return option. For example, this returns the 10 most relevant messages: C: C3 SORT RETURN (PARTIAL 1:10) (RELEVANCY) UTF-8 FUZZY TEXT "World" S: * ESEARCH (TAG "C3") PARTIAL (1:10 42,9,34,13,15,4,2,7,23,82) S: C3 OK Sort completed. 7. Formal Syntax The following syntax specification uses the augmented Backus-Naur Form (BNF) as described in [ABNF]. It includes definitions from [RFC3501], [IMAP-ABNF], and [SORT]. capability =/ "SEARCH=FUZZY" score = 1*3DIGIT ;; (1 <= n <= 100) score-list = "(" [score *(SP score)] ")" search-key =/ "FUZZY" SP search-key search-return-data =/ "RELEVANCY" SP score-list ;; Conforms to , from [IMAP-ABNF] search-return-opt =/ "RELEVANCY" ;; Conforms to , from [IMAP-ABNF] sort-key =/ "RELEVANCY" 8. Security Considerations Implementation of this extension might enable denial-of-service attacks against server resources. Servers MAY limit the resources that a single search (or a single user) may use. Additionally, implementors should be aware of the following: Fuzzy search engines are often complex with non-obvious disk space, memory, and/or CPU usage patterns. Server implementors should at least test the fuzzy- search behavior with large messages that contain very long words and/or unique random strings. Also, very long search keys might cause excessive memory or CPU usage. Invalid input may also be problematic. For example, if the search engine takes a UTF-8 stream as input, it might fail more or less badly when illegal UTF-8 sequences are fed to it from a message whose Sirainen Standards Track [Page 5] RFC 6203 IMAP4 FUZZY Search March 2011 character set was claimed to be UTF-8. This could be avoided by validating all the input and, for example, replacing illegal UTF-8 sequences with the Unicode replacement character (U+FFFD). Search relevancy rankings might be susceptible to "poisoning" by smart attackers using certain keywords or hidden markup (e.g., HTML) in their messages to boost the rankings. This can't be fully prevented by servers, so clients should prepare for it by at least allowing users to see all the search results, rather than hiding results below a certain score. 9. IANA Considerations IMAP4 capabilities are registered by publishing a standards track or IESG-approved experimental RFC. The "Internet Message Access Protocol (IMAP) 4 Capabilities Registry" is available from http://www.iana.org/. This document defines the SEARCH=FUZZY IMAP capability. IANA has added it to the registry. 10. Acknowledgements Alexey Melnikov, Zoltan Ordogh, Barry Leiba, Cyrus Daboo, and Dave Cridland have helped with this document. 11. Normative References [ABNF] Crocker, D., Ed. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [CONTEXT] Cridland, D. and C. King, "Contexts for IMAP4", RFC 5267, July 2008. [ESEARCH] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH Command for Controlling What Kind of Information Is Returned", RFC 4731, November 2006. [IMAP-ABNF] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. Sirainen Standards Track [Page 6] RFC 6203 IMAP4 FUZZY Search March 2011 [RFC5255] Newman, C., Gulbrandsen, A., and A. Melnikov, "Internet Message Access Protocol Internationalization", RFC 5255, June 2008. [SORT] Crispin, M. and K. Murchison, "Internet Message Access Protocol - SORT and THREAD Extensions", RFC 5256, June 2008. Author's Address Timo Sirainen EMail: tss@iki.fi Sirainen Standards Track [Page 7] ================================================ FILE: docs/rfcs/rfc6237.IMAP4_Multimailbox_SEARCH_extension.txt ================================================ Internet Engineering Task Force (IETF) B. Leiba Request for Comments: 6237 Huawei Technologies Updates: 4466 A. Melnikov Category: Experimental Isode Limited ISSN: 2070-1721 May 2011 IMAP4 Multimailbox SEARCH Extension Abstract The IMAP4 specification allows the searching of only the selected mailbox. A user often wants to search multiple mailboxes, and a client that wishes to support this must issue a series of SELECT and SEARCH commands, waiting for each to complete before moving on to the next. This extension allows a client to search multiple mailboxes with one command, limiting the round trips and waiting for various searches to complete, and not requiring disruption of the currently selected mailbox. This extension also uses MAILBOX and TAG fields in ESEARCH responses, allowing a client to pipeline the searches if it chooses. This document updates RFC 4466. Status of This Memo This document is not an Internet Standards Track specification; it is published for examination, experimental implementation, and evaluation. This document defines an Experimental Protocol for the Internet community. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Not all documents approved by the IESG are a candidate for any level of Internet Standard; see Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc6237. Leiba & Melnikov Experimental [Page 1] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 Copyright Notice Copyright (c) 2011 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Table of Contents 1. Introduction ....................................................2 1.1. Conventions Used in This Document ..........................3 2. New ESEARCH Command .............................................3 2.1. The ESEARCH Response .......................................4 2.2. Source Options: Specifying Mailboxes to Search .............5 3. Examples ........................................................6 4. Formal Syntax ...................................................7 5. Security Considerations .........................................8 6. IANA Considerations .............................................9 7. Acknowledgements ................................................9 8. Normative References ............................................9 1. Introduction The IMAP4 specification allows the searching of only the selected mailbox. A user often wants to search multiple mailboxes, and a client that wishes to support this must issue a series of SELECT and SEARCH commands, waiting for each to complete before moving on to the next. The commands can't be pipelined, because the server might run them in parallel, and the untagged SEARCH responses could not then be distinguished from each other. This extension allows a client to search multiple mailboxes with one command, and includes MAILBOX and TAG fields in the ESEARCH response, yielding the following advantages: o A single command limits the number of round trips needed to search a set of mailboxes. o A single command eliminates the need to wait for one search to complete before starting the next. Leiba & Melnikov Experimental [Page 2] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 o A single command allows the server to optimize the search, if it can. o A command that is not dependent upon the selected mailbox eliminates the need to disrupt the selection state or to open another IMAP connection. o The MAILBOX, UIDVALIDITY, and TAG fields in the responses allow a client to distinguish which responses go with which search (and which mailbox). A client can safely pipeline these search commands without danger of confusion. The addition of the MAILBOX and UIDVALIDITY fields updates the search-correlator item defined in [RFC4466]. 1.1. Conventions Used in This Document In examples, "C:" indicates lines sent by a client that is connected to a server. "S:" indicates lines sent by the server to the client. The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 [RFC2119]. 2. New ESEARCH Command Arguments: OPTIONAL source options OPTIONAL result options OPTIONAL charset specification (see [RFC2978]) searching criteria (one or more) Responses: REQUIRED untagged response: ESEARCH Result: OK -- search completed NO -- error: cannot search that charset or criteria BAD -- command unknown or arguments invalid This section defines a new ESEARCH command, which works similarly to the UID SEARCH command described in Section 2.6.1 of [RFC4466] (initially described in Section 6.4.4 of [RFC3501] and extended by [RFC4731]). The ESEARCH command further extends searching by allowing for optional source and result options. This document does not define any new result options (see Section 3.1 of [RFC4731]). A server that supports this extension includes "MULTISEARCH" in its IMAP capability string. Leiba & Melnikov Experimental [Page 3] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 Because there has been confusion about this, it is worth pointing out that with ESEARCH, as with *any* SEARCH or UID SEARCH command, it MUST NOT be considered an error if the search terms include a range of message numbers that extends (or, in fact, starts) beyond the end of the mailbox. For example, a client might want to establish a rolling window through the search results this way: C: tag1 UID ESEARCH FROM "frobozz" 1:100 ...followed later by this: C: tag1 UID ESEARCH FROM "frobozz" 101:200 ...and so on. This tells the server to match only the first hundred messages in the mailbox the first time, the second hundred the second time, etc. In fact, it might likely allow the server to optimize the search significantly. In the above example, whether the mailbox contains 50 or 150 or 250 messages, neither of the search commands shown will result in an error. It is up to the client to know when to stop moving its search window. 2.1. The ESEARCH Response In response to an ESEARCH command, the server MUST return ESEARCH responses [RFC4731] (that is, not SEARCH responses). Because message numbers are not useful for mailboxes that are not selected, the responses MUST contain information about UIDs, not message numbers. This is true even if the source options specify that only the selected mailbox be searched. Presence of a source option in the absence of a result option implies the "ALL" result option (see Section 3.1 of [RFC4731]). Note that this is not the same as the result from the SEARCH command described in the IMAP base protocol [RFC3501]. Source options describe which mailboxes must be searched for messages. An ESEARCH command with source options does not affect which mailbox, if any, is currently selected, regardless of which mailboxes are searched. For each mailbox satisfying the source options, a single ESEARCH response MUST be returned if any messages in that mailbox match the search criteria. An ESEARCH response MUST NOT be returned for mailboxes that contain no matching messages. This is true even when result options such as MIN, MAX, and COUNT are specified (see Section 3.1 of [RFC4731]), and the values returned (lowest UID matched, highest UID matched, and number of messages matched, respectively) apply to the mailbox reported in that ESEARCH response. Leiba & Melnikov Experimental [Page 4] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 Note that it is possible for an ESEARCH command to return *no* untagged responses (no ESEARCH responses at all), in the case that there are no matches to the search in any of the mailboxes that satisfy the source options. Clients can detect this situation by finding the tagged OK response without having received any matching untagged ESEARCH responses. Each ESEARCH response MUST contain the MAILBOX, TAG, and UIDVALIDITY correlators. Correlators allow clients to issue several ESEARCH commands at once (pipelined). If the SEARCHRES [RFC5182] extension is used in an ESEARCH command, that ESEARCH command MUST be executed by the server after all previous SEARCH/ESEARCH commands have completed and before any subsequent SEARCH/ESEARCH commands are executed. The server MAY perform consecutive ESEARCH commands in parallel as long as none of them use the SEARCHRES extension. 2.2. Source Options: Specifying Mailboxes to Search The source options, if present, MUST contain a mailbox specifier as defined in the IMAP NOTIFY extension [RFC5465], Section 6 (using the "filter-mailboxes" ABNF item), with the following differences: 1. The "selected-delayed" specifier is not valid here. 2. A "subtree-one" specifier is added. The "subtree" specifier results in a search of the specified mailbox and all selectable mailboxes that are subordinate to it, through an indefinitely deep hierarchy. The "subtree-one" specifier results in a search of the specified mailbox and all selectable child mailboxes, one hierarchy level down. If "subtree" is specified, the server MUST defend against loops in the hierarchy (for example, those caused by recursive file-system links within the message store). The server SHOULD do this by keeping track of the mailboxes that have been searched, and terminating the hierarchy traversal when a repeat is found. If it cannot do that, it MAY do it by limiting the hierarchy depth. If the source options are not present, the value "selected" is assumed -- that is, only the currently selected mailbox is searched. The "personal" source option is a particularly convenient way to search all of the current user's mailboxes. Note that there is no way to use wildcard characters to search all mailboxes; the "mailboxes" source option does not do wildcard expansion. Leiba & Melnikov Experimental [Page 5] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 If the source options include (or default to) "selected", the IMAP session MUST be in "selected" state. If the source options specify other mailboxes and NOT "selected", then the IMAP session MUST be in either "selected" or "authenticated" state. If the session is not in a correct state, the ESEARCH command MUST return a "BAD" result. If the server supports the SEARCHRES [RFC5182] extension, then the "SAVE" result option is valid *only* if "selected" is specified or defaulted as the sole mailbox to be searched. If any source option other than "selected" is specified, the ESEARCH command MUST return a "BAD" result. If the server supports the CONTEXT=SEARCH and/or CONTEXT=SORT extension [RFC5267], then the following additional rules apply: o The CONTEXT return option (Section 4.2 of [RFC5267]) can be used with an ESEARCH command. o If the UPDATE return option is used (Section 4.3 of [RFC5267]), it MUST apply ONLY to the currently selected mailbox. If UPDATE is used and there is no mailbox currently selected, the ESEARCH command MUST return a "BAD" result. o The PARTIAL search return option (Section 4.4 of [RFC5267]) can be used and applies to each mailbox searched by the ESEARCH command. If the server supports the Access Control List (ACL) [RFC4314] extension, then the logged-in user is required to have the "r" right for each mailbox she wants to search. In addition, any mailboxes that are not explicitly named (accessed through "personal" or "subtree", for example) are required to have the "l" right. Mailboxes matching the source options for which the logged-in user lacks sufficient rights MUST be ignored by the ESEARCH command processing. In particular, ESEARCH responses MUST NOT be returned for those mailboxes. 3. Examples In the following example, note that two ESEARCH commands are pipelined, and that the server is running them in parallel, interleaving a response to the second search amid the responses to the first (watch the tags). C: tag1 ESEARCH IN (mailboxes "folder1" subtree "folder2") unseen C: tag2 ESEARCH IN (mailboxes "folder1" subtree-one "folder2") subject "chad" S: * ESEARCH (TAG "tag1" MAILBOX "folder1" UIDVALIDITY 1) UID ALL 4001,4003,4005,4007,4009 Leiba & Melnikov Experimental [Page 6] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 S: * ESEARCH (TAG "tag2" MAILBOX "folder1" UIDVALIDITY 1) UID ALL 3001:3004,3788 S: * ESEARCH (TAG "tag1" MAILBOX "folder2/banana" UIDVALIDITY 503) UID ALL 3002,4004 S: * ESEARCH (TAG "tag1" MAILBOX "folder2/peach" UIDVALIDITY 3) UID ALL 921691 S: tag1 OK done S: * ESEARCH (TAG "tag2" MAILBOX "folder2/salmon" UIDVALIDITY 1111111) UID ALL 50003,50006,50009,50012 S: tag2 OK done 4. Formal Syntax The following syntax specification uses the Augmented Backus-Naur Form (ABNF) as described in [RFC5234]. Terms not defined here are taken from [RFC3501], [RFC5465], or [RFC4466]. command-auth =/ esearch ; Update definition from IMAP base [RFC3501]. ; Add new "esearch" command. command-select =/ esearch ; Update definition from IMAP base [RFC3501]. ; Add new "esearch" command. filter-mailboxes-other =/ ("subtree-one" SP one-or-more-mailbox) ; Update definition from IMAP Notify [RFC5465]. ; Add new "subtree-one" selector. filter-mailboxes-selected = "selected" ; Update definition from IMAP Notify [RFC5465]. ; We forbid the use of "selected-delayed". one-correlator = ("TAG" SP tag-string) / ("MAILBOX" SP astring) / ("UIDVALIDITY" SP nz-number) ; Each correlator MUST appear exactly once. scope-option = scope-option-name [SP scope-option-value] ; No options defined here. Syntax for future extensions. scope-option-name = tagged-ext-label ; No options defined here. Syntax for future extensions. scope-option-value = tagged-ext-val ; No options defined here. Syntax for future extensions. Leiba & Melnikov Experimental [Page 7] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 scope-options = scope-option *(SP scope-option) ; A given option may only appear once. ; No options defined here. Syntax for future extensions. esearch = "ESEARCH" [SP esearch-source-opts] [SP search-return-opts] SP search-program search-correlator = SP "(" one-correlator *(SP one-correlator) ")" ; Updates definition in IMAP4 ABNF [RFC4466]. esearch-source-opts = "IN" SP "(" source-mbox [SP "(" scope-options ")"] ")" source-mbox = filter-mailboxes *(SP filter-mailboxes) ; "filter-mailboxes" is defined in IMAP Notify [RFC5465]. ; See updated definition of filter-mailboxes-other, above. ; See updated definition of filter-mailboxes-selected, above. 5. Security Considerations This new IMAP ESEARCH command allows a single command to search many mailboxes at once. On the one hand, a client could do that by sending many IMAP SEARCH commands. On the other hand, this makes it easier for a client to overwork a server, by sending a single command that results in an expensive search of tens of thousands of mailboxes. Server implementations need to be aware of that, and provide mechanisms that prevent a client from adversely affecting other users. Limitations on the number of mailboxes that may be searched in one command, and/or on the server resources that will be devoted to responding to a single client, are reasonable limitations for an implementation to impose. Implementations MUST, of course, apply access controls appropriately, limiting a user's access to ESEARCH in the same way its access is limited for any other IMAP commands. This extension has no data- access risks beyond what may be there in the unextended IMAP implementation. Mailboxes matching the source options for which the logged-in user lacks sufficient rights MUST be ignored by the ESEARCH command processing (see the paragraph about this in Section 2.2). In particular, any attempt to distinguish insufficient access from non-existent mailboxes may expose information about the mailbox hierarchy that isn't otherwise available to the client. If "subtree" is specified, the server MUST defend against loops in the hierarchy (see the paragraph about this in Section 2.2). Leiba & Melnikov Experimental [Page 8] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 6. IANA Considerations IMAP4 capabilities are registered by publishing a Standards Track or IESG-approved Experimental RFC. The "IMAP 4 Capabilities" registry is currently located here: http://www.iana.org/ This document defines the IMAP capability "MULTISEARCH", and IANA has added it to the registry. 7. Acknowledgements The authors gratefully acknowledge feedback provided by Timo Sirainen, Peter Coates, and Arnt Gulbrandsen. 8. Normative References [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. [RFC2978] Freed, N. and J. Postel, "IANA Charset Registration Procedures", BCP 19, RFC 2978, October 2000. [RFC3501] Crispin, M., "INTERNET MESSAGE ACCESS PROTOCOL - VERSION 4rev1", RFC 3501, March 2003. [RFC4314] Melnikov, A., "IMAP4 Access Control List (ACL) Extension", RFC 4314, December 2005. [RFC4466] Melnikov, A. and C. Daboo, "Collected Extensions to IMAP4 ABNF", RFC 4466, April 2006. [RFC4731] Melnikov, A. and D. Cridland, "IMAP4 Extension to SEARCH Command for Controlling What Kind of Information Is Returned", RFC 4731, November 2006. [RFC5182] Melnikov, A., "IMAP Extension for Referencing the Last SEARCH Result", RFC 5182, March 2008. [RFC5234] Crocker, D., Ed., and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5267] Cridland, D. and C. King, "Contexts for IMAP4", RFC 5267, July 2008. Leiba & Melnikov Experimental [Page 9] RFC 6237 IMAP4 Multimailbox SEARCH Extension May 2011 [RFC5465] Gulbrandsen, A., King, C., and A. Melnikov, "The IMAP NOTIFY Extension", RFC 5465, February 2009. Authors' Addresses Barry Leiba Huawei Technologies Phone: +1 646 827 0648 EMail: barryleiba@computer.org URI: http://internetmessagingtechnology.org/ Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Leiba & Melnikov Experimental [Page 10] ================================================ FILE: docs/rfcs/rfc6331.Moving_Digest-MD5_to_Historic ================================================ Internet Engineering Task Force (IETF) A. Melnikov Request for Comments: 6331 Isode Limited Obsoletes: 2831 July 2011 Category: Informational ISSN: 2070-1721 Moving DIGEST-MD5 to Historic Abstract This memo describes problems with the DIGEST-MD5 Simple Authentication and Security Layer (SASL) mechanism as specified in RFC 2831. It marks DIGEST-MD5 as OBSOLETE in the IANA Registry of SASL mechanisms and moves RFC 2831 to Historic status. Status of This Memo This document is not an Internet Standards Track specification; it is published for informational purposes. This document is a product of the Internet Engineering Task Force (IETF). It represents the consensus of the IETF community. It has received public review and has been approved for publication by the Internet Engineering Steering Group (IESG). Not all documents approved by the IESG are a candidate for any level of Internet Standard; see Section 2 of RFC 5741. Information about the current status of this document, any errata, and how to provide feedback on it may be obtained at http://www.rfc-editor.org/info/rfc6331. Copyright Notice Copyright (c) 2011 IETF Trust and the persons identified as the document authors. All rights reserved. This document is subject to BCP 78 and the IETF Trust's Legal Provisions Relating to IETF Documents (http://trustee.ietf.org/license-info) in effect on the date of publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. Melnikov Informational [Page 1] RFC 6331 Moving DIGEST-MD5 to Historic July 2011 This document may contain material from IETF Documents or IETF Contributions published or made publicly available before November 10, 2008. The person(s) controlling the copyright in some of this material may not have granted the IETF Trust the right to allow modifications of such material outside the IETF Standards Process. Without obtaining an adequate license from the person(s) controlling the copyright in such materials, this document may not be modified outside the IETF Standards Process, and derivative works of it may not be created outside the IETF Standards Process, except to format it for publication as an RFC or to translate it into languages other than English. Table of Contents 1. Introduction and Overview . . . . . . . . . . . . . . . . . . 2 2. Security Considerations . . . . . . . . . . . . . . . . . . . 5 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 5 4. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 5 5. References . . . . . . . . . . . . . . . . . . . . . . . . . 5 5.1. Normative References . . . . . . . . . . . . . . . . . . . . 5 5.2. Informative References . . . . . . . . . . . . . . . . . . . 5 1. Introduction and Overview [RFC2831] defines how HTTP Digest Authentication [RFC2617] can be used as a Simple Authentication and Security Layer (SASL) [RFC4422] mechanism for any protocol that has a SASL profile. It was intended both as an improvement over CRAM-MD5 [RFC2195] and as a convenient way to support a single authentication mechanism for web, email, the Lightweight Directory Access Protocol (LDAP), and other protocols. While it can be argued that it is an improvement over CRAM-MD5, many implementors commented that the additional complexity of DIGEST-MD5 makes it difficult to implement fully and securely. Below is an incomplete list of problems with the DIGEST-MD5 mechanism as specified in [RFC2831]: 1. The mechanism has too many options and modes. Some of them are not well described and are not widely implemented. For example, DIGEST-MD5 allows the "qop" directive to contain multiple values, but it also allows for multiple qop directives to be specified. The handling of multiple options is not specified, which results in minor interoperability problems. Some implementations amalgamate multiple qop values into one, while others treat multiple qops as an error. Another example is the use of an empty authorization identity. In SASL, an empty authorization identity means that the client is willing to authorize as the authentication identity. The document is not clear on whether Melnikov Informational [Page 2] RFC 6331 Moving DIGEST-MD5 to Historic July 2011 the authzid must be omitted or if it can be specified with an empty value to convey this. The requirement for backward compatibility with HTTP Digest means that the situation is even worse. For example, DIGEST-MD5 requires all usernames/passwords that can be entirely represented in the ISO-8859-1 charset to be down converted from UTF-8 [RFC3629] to ISO-8859-1 [ISO-8859-1]. Another example is the use of quoted strings. Handling of characters that need escaping is not properly described, and the DIGEST-MD5 document has no examples to demonstrate correct behavior. 2. The DIGEST-MD5 document uses ABNF from RFC 822 [RFC0822], which allows an extra construct and allows for "implied folding whitespace" to be inserted in many places. The difference from a more common ABNF defined in [RFC5234] is confusing for some implementors. As a result, many implementations do not accept folding whitespace in many places where it is allowed. 3. The DIGEST-MD5 document uses the concept of a "realm" to define a collection of accounts. A DIGEST-MD5 server can support one or more realms. The DIGEST-MD5 document does not provide any guidance on how realms should be named and, more importantly, how they can be entered in User Interfaces (UIs). As a result, many DIGEST-MD5 clients have confusing UIs, do not allow users to enter a realm, and/or do not allow users to pick one of the server-supported realms. 4. Use of username in the inner hash is problematic. The inner hash of DIGEST-MD5 is an MD5 hash of colon-separated username, realm, and password. Implementations may choose to store inner hashes instead of clear text passwords. This has some useful properties, such as protection from compromise of authentication databases containing the same username and password on other servers if a server with the username and password is compromised; however, this is rarely done in practice. First, the inner hash is not compatible with widely deployed Unix password databases, and second, changing the username would invalidate the inner hash. 5. Description of DES/3DES [DES] and RC4 security layers are inadequate to produce independently developed interoperable implementations. In the DES/3DES case, this is partly a problem with existing DES APIs. 6. DIGEST-MD5 outer hash (the value of the "response" directive) does not protect the whole authentication exchange, which makes the mechanism vulnerable to "man-in-the-middle" (MITM) attacks, such as modification of the list of supported qops or ciphers. Melnikov Informational [Page 3] RFC 6331 Moving DIGEST-MD5 to Historic July 2011 7. The following features are missing from DIGEST-MD5, making it insecure or unsuitable for use in protocols: A. Channel bindings [RFC5056]. B. Hash agility (i.e., no easy way to replace the MD5 hash function with another one). C. Support for SASLPrep [RFC4013] or any other type of Unicode character normalization of usernames and passwords. The original DIGEST-MD5 document predates SASLPrep and does not recommend any Unicode character normalization. 8. The cryptographic primitives in DIGEST-MD5 are not up to today's standards, in particular: A. The MD5 hash is sufficiently weak to make a brute force attack on DIGEST-MD5 easy with common hardware [RFC6151]. B. The RC4 algorithm is prone to attack when used as the security layer without discarding the initial key stream output [RFC6229]. C. The DES cipher for the security layer is considered insecure due to its small key space [RFC3766]. Note that most of the problems listed above are already present in the HTTP Digest authentication mechanism. Because DIGEST-MD5 is defined as an extensible mechanism, it is possible to fix most of the problems listed above. However, this would increase implementation complexity of an already complex mechanism even further, so the effort is not worth the cost. In addition, an implementation of a "fixed" DIGEST-MD5 specification would likely either not interoperate with any existing implementation of [RFC2831] or would be vulnerable to various downgrade attacks. Note that despite DIGEST-MD5 seeing some deployment on the Internet, this specification recommends obsoleting DIGEST-MD5 because DIGEST- MD5, as implemented, is not a reasonable candidate for further standardization and should be deprecated in favor of one or more new password-based mechanisms currently being designed. The Salted Challenge Response Authentication Mechanism (SCRAM) family of SASL mechanisms [RFC5802] has been developed to provide similar features as DIGEST-MD5 but with a better design. Melnikov Informational [Page 4] RFC 6331 Moving DIGEST-MD5 to Historic July 2011 2. Security Considerations Security issues are discussed throughout this document. 3. IANA Considerations IANA has changed the "Intended usage" of the DIGEST-MD5 mechanism registration in the SASL mechanism registry to OBSOLETE. The SASL mechanism registry is specified in [RFC4422] and is currently available at: http://www.iana.org/assignments/sasl-mechanisms 4. Acknowledgements The author gratefully acknowledges the feedback provided by Chris Newman, Simon Josefsson, Kurt Zeilenga, Sean Turner, and Abhijit Menon-Sen. Various text was copied from other RFCs, in particular, from [RFC2831]. 5. References 5.1. Normative References [RFC2617] Franks, J., Hallam-Baker, P., Hostetler, J., Lawrence, S., Leach, P., Luotonen, A., and L. Stewart, "HTTP Authentication: Basic and Digest Access Authentication", RFC 2617, June 1999. [RFC2831] Leach, P. and C. Newman, "Using Digest Authentication as a SASL Mechanism", RFC 2831, May 2000. 5.2. Informative References [DES] National Institute of Standards and Technology, "Data Encryption Standard (DES)", FIPS PUB 46-3, October 1999. [ISO-8859-1] International Organization for Standardization, "Information technology - 8-bit single-byte coded graphic character sets - Part 1: Latin alphabet No. 1", ISO/IEC 8859-1, 1998. [RFC0822] Crocker, D., "Standard for the format of ARPA Internet text messages", STD 11, RFC 822, August 1982. Melnikov Informational [Page 5] RFC 6331 Moving DIGEST-MD5 to Historic July 2011 [RFC2195] Klensin, J., Catoe, R., and P. Krumviede, "IMAP/POP AUTHorize Extension for Simple Challenge/Response", RFC 2195, September 1997. [RFC3629] Yergeau, F., "UTF-8, a transformation format of ISO 10646", STD 63, RFC 3629, November 2003. [RFC3766] Orman, H. and P. Hoffman, "Determining Strengths For Public Keys Used For Exchanging Symmetric Keys", BCP 86, RFC 3766, April 2004. [RFC4013] Zeilenga, K., "SASLprep: Stringprep Profile for User Names and Passwords", RFC 4013, February 2005. [RFC4422] Melnikov, A. and K. Zeilenga, "Simple Authentication and Security Layer (SASL)", RFC 4422, June 2006. [RFC5056] Williams, N., "On the Use of Channel Bindings to Secure Channels", RFC 5056, November 2007. [RFC5234] Crocker, D. and P. Overell, "Augmented BNF for Syntax Specifications: ABNF", STD 68, RFC 5234, January 2008. [RFC5802] Newman, C., Menon-Sen, A., Melnikov, A., and N. Williams, "Salted Challenge Response Authentication Mechanism (SCRAM) SASL and GSS-API Mechanisms", RFC 5802, July 2010. [RFC6151] Turner, S. and L. Chen, "Updated Security Considerations for the MD5 Message-Digest and the HMAC- MD5 Algorithms", RFC 6151, March 2011. [RFC6229] Strombergson, J. and S. Josefsson, "Test Vectors for the Stream Cipher RC4", RFC 6229, May 2011. Author's Address Alexey Melnikov Isode Limited 5 Castle Business Village 36 Station Road Hampton, Middlesex TW12 2BX UK EMail: Alexey.Melnikov@isode.com URI: http://www.melnikov.ca/ Melnikov Informational [Page 6] ================================================ FILE: docs/website-doc.sh ================================================ #!/bin/sh # # vim: expandtab ts=2 : ARGS=$* SPHINXBUILD=sphinx-build TMPDIR='/tmp/offlineimap-sphinx-doctrees' WEBSITE='./website' DOCBASE="${WEBSITE}/_doc" DESTBASE="${DOCBASE}/versions" VERSIONS_YML="${WEBSITE}/_data/versions.yml" ANNOUNCES_YML="${WEBSITE}/_data/announces.yml" ANNOUNCES_YML_LIMIT=31 ANNOUNCES_YML_TMP="${ANNOUNCES_YML}.tmp" CONTRIB_YML="${WEBSITE}/_data/contribs.yml" CONTRIB="${DOCBASE}/contrib" HEADER="# DO NOT EDIT MANUALLY: it is generated by a script (website-doc.sh)." function fix_pwd () { cd "$(git rev-parse --show-toplevel)" || \ exit 2 "cannot determine the root of the repository" test -d "$DESTBASE" || exit 1 } fix_pwd version="v$(./offlineimap.py --version)" # # Add the doc for the contrib files. # function contrib () { echo $HEADER > "$CONTRIB_YML" # systemd cp -afv "./contrib/systemd/README.md" "${CONTRIB}/systemd.md" echo "- {filename: 'systemd', linkname: 'Integrate with systemd'}" >> "$CONTRIB_YML" } # # Build the sphinx documentation. # function api () { # Build the doc with sphinx. dest="${DESTBASE}/${version}" echo "Cleaning target directory: $dest" rm -rf "$dest" $SPHINXBUILD -b html -d "$TMPDIR" ./docs/doc-src "$dest" # Build the JSON definitions for Jekyll. # This let know the website about the available APIs documentations. echo "Building Jekyll data: $VERSIONS_YML" # Erase previous content. echo > "$VERSIONS_YML" <> "$VERSIONS_YML" } # # Return title from release entry. # $1: full release title # function parse_releases_get_link () { echo $1 | sed -r -e 's,^### (OfflineIMAP.*)\),\1,' \ | tr '[:upper:]' '[:lower:]' \ | sed -r -e 's,[\.("],,g' \ | sed -r -e 's, ,-,g' } # # Return version from release entry. # $1: full release title # function parse_releases_get_version () { echo $1 | sed -r -e 's,^### [a-Z]+ (v[^ ]+).*,\1,' } # # Return date from release entry. # $1: full release title # function parse_releases_get_date () { echo $1 | sed -r -e 's,.*\(([0-9]+-[0-9]+-[0-9]+).*,\1,' } # # Make Changelog public and save links to them as JSON. # function releases () { # Copy the Changelogs. for foo in ./Changelog.md ./Changelog.maint.md do cp -afv "$foo" "$DOCBASE" done # Build the announces JSON list. Format is JSON: # - {version: '', link: ''} # - ... echo "$HEADER" > "$ANNOUNCES_YML" # Announces for the mainline. grep -E '^### OfflineIMAP' ./Changelog.md | while read title do link="$(parse_releases_get_link "$title")" v="$(parse_releases_get_version "$title")" d="$(parse_releases_get_date "$title")" echo "- {date: '${d}', version: '${v}', link: 'Changelog.html#${link}'}" done | tee -a "$ANNOUNCES_YML_TMP" # Announces for the maintenance releases. grep -E '^### OfflineIMAP' ./Changelog.maint.md | while read title do link="$(parse_releases_get_link "$title")" v="$(parse_releases_get_version "$title")" d="$(parse_releases_get_date "$title")" echo "- {date: '${d}', version: '${v}', link: 'Changelog.maint.html#${link}'}" done | tee -a "$ANNOUNCES_YML_TMP" sort -nr "$ANNOUNCES_YML_TMP" | head -n $ANNOUNCES_YML_LIMIT >> "$ANNOUNCES_YML" rm -f "$ANNOUNCES_YML_TMP" } function manhtml () { set -e cd ./docs make manhtml cd .. cp -afv ./docs/manhtml/* "$DOCBASE" } exit_code=0 test "n$ARGS" = 'n' && ARGS='usage' # no option passed for arg in $ARGS do # PWD was fixed at the very beginning. case "n$arg" in "nreleases") releases ;; "napi") api ;; "nhtml") manhtml ;; "ncontrib") contrib ;; "nusage") echo "Usage: website-doc.sh " ;; *) echo "unkown option $arg" exit_code=$(( $exit_code + 1 )) ;; esac done exit $exit_code ================================================ FILE: offlineimap/CustomConfig.py ================================================ # Copyright (C) 2003-2016 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import re from sys import exc_info import six try: from ConfigParser import SafeConfigParser, Error except ImportError: # Python3. from configparser import SafeConfigParser, Error from offlineimap.localeval import LocalEval class CustomConfigParser(SafeConfigParser): def __init__(self): SafeConfigParser.__init__(self) self.localeval = None def getdefault(self, section, option, default, *args, **kwargs): """Same as config.get, but returns the value of `default` if there is no such option specified.""" if self.has_option(section, option): return self.get(*(section, option) + args, **kwargs) else: return default def getdefaultint(self, section, option, default, *args, **kwargs): """Same as config.getint, but returns the value of `default` if there is no such option specified.""" if self.has_option(section, option): return self.getint(*(section, option) + args, **kwargs) else: return default def getdefaultfloat(self, section, option, default, *args, **kwargs): """Same as config.getfloat, but returns the value of `default` if there is no such option specified.""" if self.has_option(section, option): return self.getfloat(*(section, option) + args, **kwargs) else: return default def getdefaultboolean(self, section, option, default, *args, **kwargs): """Same as config.getboolean, but returns the value of `default` if there is no such option specified.""" if self.has_option(section, option): return self.getboolean(*(section, option) + args, **kwargs) else: return default def getlist(self, section, option, separator_re): """Parses option as the list of values separated by the given regexp.""" try: val = self.get(section, option).strip() return re.split(separator_re, val) except re.error as e: six.reraise(Error, Error("Bad split regexp '%s': %s"% (separator_re, e)), exc_info()[2]) def getdefaultlist(self, section, option, default, separator_re): """Same as getlist, but returns the value of `default` if there is no such option specified.""" if self.has_option(section, option): return self.getlist(*(section, option, separator_re)) else: return default def getmetadatadir(self): xforms = [os.path.expanduser, os.path.expandvars] d = self.getdefault("general", "metadata", "~/.offlineimap") metadatadir = self.apply_xforms(d, xforms) if not os.path.exists(metadatadir): os.mkdir(metadatadir, 0o700) return metadatadir def getlocaleval(self): # We already loaded pythonfile, so return this copy. if self.localeval is not None: return self.localeval xforms = [os.path.expanduser, os.path.expandvars] if self.has_option("general", "pythonfile"): path = self.get("general", "pythonfile") path = self.apply_xforms(path, xforms) else: path = None self.localeval = LocalEval(path) return self.localeval def getsectionlist(self, key): """Returns a list of sections that start with (str) key + " ". That is, if key is "Account", returns all section names that start with "Account ", but strips off the "Account ". For instance, for "Account Test", returns "Test".""" key = key + ' ' return [x[len(key):] for x in self.sections() \ if x.startswith(key)] def set_if_not_exists(self, section, option, value): """Set a value if it does not exist yet. This allows to set default if the user has not explicitly configured anything.""" if not self.has_option(section, option): self.set(section, option, value) def apply_xforms(self, string, transforms): """Applies set of transformations to a string. Arguments: - string: source string; if None, then no processing will take place. - transforms: iterable that returns transformation function on each turn. Returns transformed string.""" if string == None: return None for f in transforms: string = f(string) return string def CustomConfigDefault(): """Just a constant that won't occur anywhere else. This allows us to differentiate if the user has passed in any default value to the getconf* functions in ConfigHelperMixin derived classes.""" pass class ConfigHelperMixin(object): """Allow comfortable retrieving of config values pertaining to a section. If a class inherits from cls:`ConfigHelperMixin`, it needs to provide 2 functions: - meth:`getconfig` (returning a CustomConfigParser object) - and meth:`getsection` (returning a string which represents the section to look up). All calls to getconf* will then return the configuration values for the CustomConfigParser object in the specific section. """ def _confighelper_runner(self, option, default, defaultfunc, mainfunc, *args): """Returns configuration or default value for option that contains in section identified by getsection(). Arguments: - option: name of the option to retrieve; - default: governs which function we will call. * When CustomConfigDefault is passed, we will call the mainfunc. * When any other value is passed, we will call the defaultfunc and the value of `default` will be passed as the third argument to this function. - defaultfunc and mainfunc: processing helpers. - args: additional trailing arguments that will be passed to all processing helpers. """ lst = [self.getsection(), option] if default == CustomConfigDefault: return mainfunc(*(lst + list(args))) else: lst.append(default) return defaultfunc(*(lst + list(args))) def getconfig(self): """Returns CustomConfigParser object that we will use for all our actions. Must be overriden in all classes that use this mix-in.""" raise NotImplementedError("ConfigHelperMixin.getconfig() " "is to be overriden") def getsection(self): """Returns name of configuration section in which our class keeps its configuration. Must be overriden in all classes that use this mix-in.""" raise NotImplementedError("ConfigHelperMixin.getsection() " "is to be overriden") def getconf(self, option, default = CustomConfigDefault): """Retrieves string from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. """ return self._confighelper_runner(option, default, self.getconfig().getdefault, self.getconfig().get) def getconf_xform(self, option, xforms, default = CustomConfigDefault): """Retrieves string from the configuration transforming the result. Arguments: - option: option name whose value is to be retrieved; - xforms: iterable that returns transform functions to be applied to the value of the option, both retrieved and default one; - default: default value for string if no such option exists. """ value = self.getconf(option, default) return self.getconfig().apply_xforms(value, xforms) def getconfboolean(self, option, default = CustomConfigDefault): """Retrieves boolean value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. """ return self._confighelper_runner(option, default, self.getconfig().getdefaultboolean, self.getconfig().getboolean) def getconfint(self, option, default = CustomConfigDefault): """ Retrieves integer value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. """ return self._confighelper_runner(option, default, self.getconfig().getdefaultint, self.getconfig().getint) def getconffloat(self, option, default = CustomConfigDefault): """Retrieves floating-point value from the configuration. Arguments: - option: option name whose value is to be retrieved; - default: default return value if no such option exists. """ return self._confighelper_runner(option, default, self.getconfig().getdefaultfloat, self.getconfig().getfloat) def getconflist(self, option, separator_re, default = CustomConfigDefault): """Retrieves strings from the configuration and splits it into the list of strings. Arguments: - option: option name whose value is to be retrieved; - separator_re: regular expression for separator to be used for split operation; - default: default return value if no such option exists. """ return self._confighelper_runner(option, default, self.getconfig().getdefaultlist, self.getconfig().getlist, separator_re) ================================================ FILE: offlineimap/__init__.py ================================================ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' # Expecting trailing "-rcN" or "" for stable releases. __version__ = "7.3.0" __copyright__ = "Copyright 2002-2019 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "offlineimap-project@lists.alioth.debian.org" __description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support" __license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)" __bigcopyright__ = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() __homepage__ = "http://www.offlineimap.org" banner = __bigcopyright__ from offlineimap.error import OfflineImapError # put this last, so we don't run into circular dependencies using # e.g. offlineimap.__version__. from offlineimap.init import OfflineImap ================================================ FILE: offlineimap/accounts.py ================================================ # Copyright (C) 2003-2016 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from subprocess import Popen, PIPE from threading import Event, Lock import os import time from sys import exc_info import traceback import six from offlineimap import mbnames, CustomConfig, OfflineImapError from offlineimap import globals from offlineimap.repository import Repository from offlineimap.ui import getglobalui from offlineimap.threadutil import InstanceLimitedThread FOLDER_NAMESPACE = 'LIMITED_FOLDER_' # Key: account name, Value: Dict of Key: remotefolder name, Value: lock. SYNC_MUTEXES = {} SYNC_MUTEXES_LOCK = Lock() try: import portalocker except: try: import fcntl except: pass # Ok if this fails, we can do without. # FIXME: spaghetti code alert! def getaccountlist(customconfig): # Account names in a list. return [name.lstrip() for name in customconfig.getsectionlist('Account')] class Account(CustomConfig.ConfigHelperMixin): """Represents an account (ie. 2 repositories) to sync. Most of the time you will actually want to use the derived :class:`accounts.SyncableAccount` which contains all functions used for syncing an account.""" # Signal gets set when we should stop looping. abort_soon_signal = Event() # Signal gets set on CTRL-C/SIGTERM. abort_NOW_signal = Event() def __init__(self, config, name): """ :param config: Representing the offlineimap configuration file. :type config: :class:`offlineimap.CustomConfig.CustomConfigParser` :param name: A (str) string denoting the name of the Account as configured. """ self.config = config self.name = name self.metadatadir = config.getmetadatadir() self.localeval = config.getlocaleval() # Store utf-8 support as a property of Account object self.utf_8_support = self.getconfboolean('utf8foldernames', False) # Current :mod:`offlineimap.ui`, can be used for logging: self.ui = getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) self.dryrun = self.config.getboolean('general', 'dry-run') self.quicknum = 0 if self.refreshperiod < 0: self.ui.warn("autorefresh for %s is negative, fixing it to 0."% name) self.refreshperiod = 0.0 if self.refreshperiod == 0.0: self.refreshperiod = None self.remoterepos = None self.localrepos = None self.statusrepos = None def getlocaleval(self): return self.localeval # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config def getname(self): return self.name def __str__(self): return self.name def getaccountmeta(self): return os.path.join(self.metadatadir, 'Account-' + self.name) # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Account ' + self.getname() @classmethod def set_abort_event(cls, config, signum): """Set skip sleep/abort event for all accounts. If we want to skip a current (or the next) sleep, or if we want to abort an autorefresh loop, the main thread can use set_abort_event() to send the corresponding signal. Signum = 1 implies that we want all accounts to abort or skip the current or next sleep phase. Signum = 2 will end the autorefresh loop, ie all accounts will return after they finished a sync. signum=3 means, abort NOW, e.g. on SIGINT or SIGTERM. This is a class method, it will send the signal to all accounts. """ if signum == 1: # resync signal, set config option for all accounts for acctsection in getaccountlist(config): config.set('Account ' + acctsection, "skipsleep", '1') elif signum == 2: # don't autorefresh anymore cls.abort_soon_signal.set() elif signum == 3: # abort ASAP cls.abort_NOW_signal.set() def get_abort_event(self): """Checks if an abort signal had been sent. If the 'skipsleep' config option for this account had been set, with `set_abort_event(config, 1)` it will get cleared in this function. Ie, we will only skip one sleep and not all. :returns: True, if the main thread had called :meth:`set_abort_event` earlier, otherwise 'False'. """ skipsleep = self.getconfboolean("skipsleep", 0) if skipsleep: self.config.set(self.getsection(), "skipsleep", '0') return skipsleep or Account.abort_soon_signal.is_set() or \ Account.abort_NOW_signal.is_set() def _sleeper(self): """Sleep if the account is set to autorefresh. :returns: 0:timeout expired, 1: canceled the timer, 2:request to abort the program, 100: if configured to not sleep at all. """ if not self.refreshperiod: return 100 kaobjs = [] if hasattr(self, 'localrepos'): kaobjs.append(self.localrepos) if hasattr(self, 'remoterepos'): kaobjs.append(self.remoterepos) for item in kaobjs: item.startkeepalive() refreshperiod = int(self.refreshperiod * 60) sleepresult = self.ui.sleep(refreshperiod, self) # Cancel keepalive for item in kaobjs: item.stopkeepalive() if sleepresult: if Account.abort_soon_signal.is_set() or \ Account.abort_NOW_signal.is_set(): return 2 self.quicknum = 0 return 1 return 0 def serverdiagnostics(self): """Output diagnostics for all involved repositories.""" remote_repo = Repository(self, 'remote') local_repo = Repository(self, 'local') #status_repo = Repository(self, 'status') self.ui.serverdiagnostics(remote_repo, 'Remote') self.ui.serverdiagnostics(local_repo, 'Local') #self.ui.serverdiagnostics(statusrepos, 'Status') def deletefolder(self, foldername): remote_repo = Repository(self, 'remote') try: if self.dryrun: self.ui.info("would try to remove '%s' on remote of '%s' " "account"% (foldername, self)) else: remote_repo.deletefolder(foldername) self.ui.info("Folder '%s' deleted."% foldername) return 0 except Exception as e: self.ui.error(e) return 1 class SyncableAccount(Account): """A syncable email account connecting 2 repositories. Derives from :class:`accounts.Account` but contains the additional functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`, used for syncing. In multi-threaded mode, one instance of this object is run per "account" thread.""" def __init__(self, *args, **kwargs): Account.__init__(self, *args, **kwargs) self._lockfd = None self._lockfilepath = os.path.join( self.config.getmetadatadir(), "%s.lock"% self) def __lock(self): """Lock the account, throwing an exception if it is locked already.""" self._lockfd = open(self._lockfilepath, 'w') try: portalocker.lock(self._lockfd, portalocker.LOCK_EX) except NameError: # portalocker not available for Windows. try: fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB) except NameError: pass # fnctl not available, disable file locking... :( except IOError: self._lockfd.close() six.reraise(OfflineImapError, OfflineImapError( "Could not lock account %s. Is another " "instance using this account?"% self, OfflineImapError.ERROR.REPO), exc_info()[2]) def _unlock(self): """Unlock the account, deleting the lock file""" #If we own the lock file, delete it if self._lockfd and not self._lockfd.closed: try: portalocker.unlock(self._lockfd) except NameError: pass self._lockfd.close() try: os.unlink(self._lockfilepath) except OSError: pass # Failed to delete for some reason. def syncrunner(self): """The target for both single and multi-threaded modes.""" self.ui.registerthread(self) try: accountmetadata = self.getaccountmeta() if not os.path.exists(accountmetadata): os.mkdir(accountmetadata, 0o700) self.remoterepos = Repository(self, 'remote') self.localrepos = Repository(self, 'local') self.statusrepos = Repository(self, 'status') except OfflineImapError as e: self.ui.error(e, exc_info()[2]) if e.severity >= OfflineImapError.ERROR.CRITICAL: raise return # Loop account sync if needed (bail out after 3 failures). looping = 3 while looping: self.ui.acct(self) try: self.__lock() self.__sync() except (KeyboardInterrupt, SystemExit): raise except OfflineImapError as e: # Stop looping and bubble up Exception if needed. if e.severity >= OfflineImapError.ERROR.REPO: if looping: looping -= 1 if e.severity >= OfflineImapError.ERROR.CRITICAL: raise self.ui.error(e, exc_info()[2]) except Exception as e: self.ui.error(e, exc_info()[2], msg= "While attempting to sync account '%s'"% self) else: # After success sync, reset the looping counter to 3. if self.refreshperiod: looping = 3 finally: self.ui.acctdone(self) self._unlock() if looping and self._sleeper() >= 2: looping = 0 def get_local_folder(self, remotefolder): """Return the corresponding local folder for a given remotefolder.""" return self.localrepos.getfolder( remotefolder.getvisiblename(). replace(self.remoterepos.getsep(), self.localrepos.getsep())) # The syncrunner will loop on this method. This means it is called more than # once during the run. def __sync(self): """Synchronize the account once, then return. Assumes that `self.remoterepos`, `self.localrepos`, and `self.statusrepos` has already been populated, so it should only be called from the :meth:`syncrunner` function.""" folderthreads = [] hook = self.getconf('presynchook', '') self.callhook(hook) if self.utf_8_support and self.remoterepos.getdecodefoldernames(): raise OfflineImapError("Configuration mismatch in account " + "'%s'. "% self.getname() + "\nAccount setting 'utf8foldernames' and repository " + "setting 'decodefoldernames'\nmay not be used at the " + "same time. This account has not been synchronized.\n" + "Please check the configuration and documentation.", OfflineImapError.ERROR.REPO) quickconfig = self.getconfint('quick', 0) if quickconfig < 0: quick = True elif quickconfig > 0: if self.quicknum == 0 or self.quicknum > quickconfig: self.quicknum = 1 quick = False else: self.quicknum = self.quicknum + 1 quick = True else: quick = False try: startedThread = False remoterepos = self.remoterepos localrepos = self.localrepos statusrepos = self.statusrepos # Init repos with list of folders, so we have them (and the # folder delimiter etc). remoterepos.getfolders() localrepos.getfolders() remoterepos.sync_folder_structure(localrepos, statusrepos) # Replicate the folderstructure between REMOTE to LOCAL. if not localrepos.getconfboolean('readonly', False): self.ui.syncfolders(remoterepos, localrepos) # Iterate through all folders on the remote repo and sync. for remotefolder in remoterepos.getfolders(): # Check for CTRL-C or SIGTERM. if Account.abort_NOW_signal.is_set(): break if not remotefolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]"% (remotefolder.getname(), remoterepos)) continue # Ignore filtered folder. # The remote folder names must not have the local sep char in # their names since this would cause troubles while converting # the name back (from local to remote). sep = localrepos.getsep() if (sep != os.path.sep and sep != remoterepos.getsep() and sep in remotefolder.getname()): self.ui.warn('', "Ignoring folder '%s' due to unsupported " "'%s' character serving as local separator."% (remotefolder.getname(), localrepos.getsep())) continue # Ignore unsupported folder name. localfolder = self.get_local_folder(remotefolder) if not localfolder.sync_this: self.ui.debug('', "Not syncing filtered folder '%s'" "[%s]"% (localfolder.getname(), localfolder.repository)) continue # Ignore filtered folder. if not globals.options.singlethreading: thread = InstanceLimitedThread( limitNamespace="%s%s"% ( FOLDER_NAMESPACE, self.remoterepos.getname()), target=syncfolder, name="Folder %s [acc: %s]"% ( remotefolder.getexplainedname(), self), args=(self, remotefolder, quick) ) thread.start() folderthreads.append(thread) else: syncfolder(self, remotefolder, quick) startedThread = True # Wait for all threads to finish. for thr in folderthreads: thr.join() if startedThread is True: mbnames.writeIntermediateFile(self.name) # Write out mailbox names. else: msg = "Account {}: no folder to sync (folderfilter issue?)".format(self) raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) localrepos.forgetfolders() remoterepos.forgetfolders() except: # Error while syncing. Drop all connections that we have, they # might be bogus by now (e.g. after suspend). localrepos.dropconnections() remoterepos.dropconnections() raise else: # Sync went fine. Hold or drop depending on config. localrepos.holdordropconnections() remoterepos.holdordropconnections() hook = self.getconf('postsynchook', '') self.callhook(hook) def callhook(self, cmd): # Check for CTRL-C or SIGTERM and run postsynchook. if Account.abort_NOW_signal.is_set(): return if not cmd: return try: self.ui.callhook("Calling hook: " + cmd) if self.dryrun: return p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) r = p.communicate() self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r) self.ui.callhook("Hook return code: %d"% p.returncode) except (KeyboardInterrupt, SystemExit): raise except Exception as e: self.ui.error(e, exc_info()[2], msg="Calling hook") #XXX: This function should likely be refactored. This should not be passed the # account instance. def syncfolder(account, remotefolder, quick): """Synchronizes given remote folder for the specified account. Filtered folders on the remote side will not invoke this function. When called in concurrently for the same localfolder, syncs are serialized.""" def acquire_mutex(): account_name = account.getname() localfolder_name = localfolder.getfullname() with SYNC_MUTEXES_LOCK: if SYNC_MUTEXES.get(account_name) is None: SYNC_MUTEXES[account_name] = {} # The localfolder full name is good to uniquely identify the sync # transaction. if SYNC_MUTEXES[account_name].get(localfolder_name) is None: #XXX: This lock could be an external file lock so we can remove # the lock at the account level. SYNC_MUTEXES[account_name][localfolder_name] = Lock() # Acquire the lock. SYNC_MUTEXES[account_name][localfolder_name].acquire() def release_mutex(): SYNC_MUTEXES[account.getname()][localfolder.getfullname()].release() def check_uid_validity(): # If either the local or the status folder has messages and # there is a UID validity problem, warn and abort. If there are # no messages, UW IMAPd loses UIDVALIDITY. But we don't really # need it if both local folders are empty. So, in that case, # just save it off. if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount() > 0: if not localfolder.check_uidvalidity(): ui.validityproblem(localfolder) localfolder.repository.restore_atime() return if not remotefolder.check_uidvalidity(): ui.validityproblem(remotefolder) localrepos.restore_atime() return else: # Both folders empty, just save new UIDVALIDITY. localfolder.save_uidvalidity() remotefolder.save_uidvalidity() def cachemessagelists_upto_date(date): """Returns messages with uid > min(uids of messages newer than date).""" remotefolder.cachemessagelist( min_date=time.gmtime(time.mktime(date) + 24*60*60)) uids = remotefolder.getmessageuidlist() localfolder.dropmessagelistcache() if len(uids) > 0: # Reload the remote message list from min_uid. This avoid issues for # old messages, which has been added from local on any previous run # (IOW, message is older than maxage _and_ has high enough UID). remotefolder.dropmessagelistcache() remotefolder.cachemessagelist(min_uid=min(uids)) localfolder.cachemessagelist(min_uid=min(uids)) else: # Remote folder UIDs list is empty for the given range. We still # might have valid local UIDs for this range (e.g.: new local # emails). localfolder.cachemessagelist(min_date=date) uids = localfolder.getmessageuidlist() # Take care to only consider positive uids. Negative UIDs might be # present due to new emails. uids = [uid for uid in uids if uid > 0] if len(uids) > 0: # Update the remote cache list for this new min(uids). remotefolder.dropmessagelistcache() remotefolder.cachemessagelist(min_uid=min(uids)) def cachemessagelists_startdate(new, partial, date): """Retrieve messagelists when startdate has been set for the folder 'partial'. Idea: suppose you want to clone the messages after date in one account (partial) to a new one (new). If new is empty, then copy messages in partial newer than date to new, and keep track of the min uid. On subsequent syncs, sync all the messages in new against those after that min uid in partial. This is a partial replacement for maxage in the IMAP-IMAP sync case, where maxage doesn't work: the UIDs of the messages in localfolder might not be in the same order as those of corresponding messages in remotefolder, so if L in local corresponds to R in remote, the ranges [L, ...] and [R, ...] might not correspond. But, if we're cloning a folder into a new one, [min_uid, ...] does correspond to [1, ...]. This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead.""" new.cachemessagelist() min_uid = partial.retrieve_min_uid() if min_uid == None: # min_uid file didn't exist if len(new.getmessageuidlist()) > 0: raise OfflineImapError("To use startdate on Repository %s, " "Repository %s must be empty"% (partial.repository.name, new.repository.name), OfflineImapError.ERROR.MESSAGE) else: partial.cachemessagelist(min_date=date) # messagelist.keys() instead of getuidmessagelist() because in # the UID mapped case we want the actual local UIDs, not their # remote counterparts. positive_uids = [uid for uid in list(partial.messagelist.keys()) if uid > 0] if len(positive_uids) > 0: min_uid = min(positive_uids) else: min_uid = 1 partial.save_min_uid(min_uid) else: partial.cachemessagelist(min_uid=min_uid) remoterepos = account.remoterepos localrepos = account.localrepos statusrepos = account.statusrepos ui = getglobalui() ui.registerthread(account) try: # Load local folder. localfolder = account.get_local_folder(remotefolder) # Acquire the mutex to start syncing. acquire_mutex() # Add the folder to the mbnames mailboxes. mbnames.add(account.name, localrepos.getlocalroot(), localfolder.getname()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename(). replace(remoterepos.getsep(), statusrepos.getsep())) statusfolder.openfiles() statusfolder.cachemessagelist() # Load local folder. ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) # Retrieve messagelists, taking into account age-restriction # options. maxage = localfolder.getmaxage() localstart = localfolder.getstartdate() remotestart = remotefolder.getstartdate() if (maxage != None) + (localstart != None) + (remotestart != None) > 1: six.reraise(OfflineImapError, OfflineImapError("You can set at most one of the " "following: maxage, startdate (for the local " "folder), startdate (for the remote folder)", OfflineImapError.ERROR.REPO), exc_info()[2]) if (maxage != None or localstart or remotestart) and quick: # IMAP quickchanged isn't compatible with options that # involve restricting the messagelist, since the "quick" # check can only retrieve a full list of UIDs in the folder. ui.warn("Quick syncs (-q) not supported in conjunction " "with maxage or startdate; ignoring -q.") if maxage != None: cachemessagelists_upto_date(maxage) check_uid_validity() elif localstart != None: cachemessagelists_startdate(remotefolder, localfolder, localstart) check_uid_validity() elif remotestart != None: cachemessagelists_startdate(localfolder, remotefolder, remotestart) check_uid_validity() else: localfolder.cachemessagelist() if quick: if (not localfolder.quickchanged(statusfolder) and not remotefolder.quickchanged(statusfolder)): ui.skippingfolder(remotefolder) localrepos.restore_atime() return check_uid_validity() remotefolder.cachemessagelist() # Synchronize remote changes. if not localrepos.getconfboolean('readonly', False): ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder) remotefolder.syncmessagesto(localfolder, statusfolder) else: ui.debug('', "Not syncing to read-only repository '%s'"% localrepos.getname()) # Synchronize local changes. if not remoterepos.getconfboolean('readonly', False): ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder) localfolder.syncmessagesto(remotefolder, statusfolder) else: ui.debug('', "Not syncing to read-only repository '%s'"% remoterepos.getname()) statusfolder.save() localrepos.restore_atime() except (KeyboardInterrupt, SystemExit): raise except OfflineImapError as e: # Bubble up severe Errors, skip folder otherwise. if e.severity > OfflineImapError.ERROR.FOLDER: raise else: ui.error(e, exc_info()[2], msg="Aborting sync, folder '%s' " "[acc: '%s']"% (localfolder, account)) except Exception as e: ui.error(e, msg="ERROR in syncfolder for %s folder %s: %s"% (account, remotefolder.getvisiblename(), traceback.format_exc())) finally: for folder in ["statusfolder", "localfolder", "remotefolder"]: if folder in locals(): locals()[folder].dropmessagelistcache() statusfolder.closefiles() # Release the mutex of this sync transaction. release_mutex() ================================================ FILE: offlineimap/bundled_imaplib2.py ================================================ #!/usr/bin/env python """Threaded IMAP4 client. Based on RFC 3501 and original imaplib module. Public classes: IMAP4 IMAP4_SSL IMAP4_stream Public functions: Internaldate2Time ParseFlags Time2Internaldate """ __all__ = ("IMAP4", "IMAP4_SSL", "IMAP4_stream", "Internaldate2Time", "ParseFlags", "Time2Internaldate", "Mon2num", "MonthNames", "InternalDate") __version__ = "2.101" __release__ = "2" __revision__ = "101" __credits__ = """ Authentication code contributed by Donn Cave June 1998. String method conversion by ESR, February 2001. GET/SETACL contributed by Anthony Baxter April 2001. IMAP4_SSL contributed by Tino Lange March 2002. GET/SETQUOTA contributed by Andreas Zeidler June 2002. PROXYAUTH contributed by Rick Holbert November 2002. IDLE via threads suggested by Philippe Normand January 2005. GET/SETANNOTATION contributed by Tomas Lindroos June 2005. COMPRESS/DEFLATE contributed by Bron Gondwana May 2009. STARTTLS from Jython's imaplib by Alan Kennedy. ID contributed by Dave Baggett November 2009. Improved untagged responses handling suggested by Dave Baggett November 2009. Improved thread naming, and 0 read detection contributed by Grant Edwards June 2010. Improved timeout handling contributed by Ivan Vovnenko October 2010. Timeout handling further improved by Ethan Glasser-Camp December 2010. Time2Internaldate() patch to match RFC2060 specification of English month names from bugs.python.org/issue11024 March 2011. starttls() bug fixed with the help of Sebastian Spaeth April 2011. Threads now set the "daemon" flag (suggested by offlineimap-project) April 2011. Single quoting introduced with the help of Vladimir Marek August 2011. Support for specifying SSL version by Ryan Kavanagh July 2013. Fix for gmail "read 0" error provided by Jim Greenleaf August 2013. Fix for offlineimap "indexerror: string index out of range" bug provided by Eygene Ryabinkin August 2013. Fix for missing idle_lock in _handler() provided by Franklin Brook August 2014. Conversion to Python3 provided by F. Malina February 2015. Fix for READ-ONLY error from multiple EXAMINE/SELECT calls by Pierre-Louis Bonicoli March 2015. Fix for null strings appended to untagged responses by Pierre-Louis Bonicoli March 2015. Fix for correct byte encoding for _CRAM_MD5_AUTH taken from python3.5 imaplib.py June 2015. Fix for correct Python 3 exception handling by Tobias Brink August 2015. Fix to allow interruptible IDLE command by Tim Peoples September 2015. Add support for TLS levels by Ben Boeckel September 2015. Fix for shutown exception by Sebastien Gross November 2015.""" __author__ = "Piers Lauder & offlineimap team" __URL__ = "http://imaplib2.sourceforge.net" __license__ = "Python License" import binascii, errno, os, random, re, select, socket, sys, time, threading, zlib if bytes != str: # Python 3, but NB assumes strings in all I/O # for backwards compatibility with python 2 usage. import queue string_types = str else: import Queue as queue string_types = basestring select_module = select # Globals CRLF = '\r\n' Debug = None # Backward compatibility IMAP4_PORT = 143 IMAP4_SSL_PORT = 993 IDLE_TIMEOUT_RESPONSE = '* IDLE TIMEOUT\r\n' IDLE_TIMEOUT = 60*29 # Don't stay in IDLE state longer READ_POLL_TIMEOUT = 30 # Without this timeout interrupted network connections can hang reader READ_SIZE = 32768 # Consume all available in socket DFLT_DEBUG_BUF_LVL = 3 # Level above which the logging output goes directly to stderr TLS_SECURE = "tls_secure" # Recognised TLS levels TLS_NO_SSL = "tls_no_ssl" TLS_COMPAT = "tls_compat" AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first # Commands CMD_VAL_STATES = 0 CMD_VAL_ASYNC = 1 NONAUTH, AUTH, SELECTED, LOGOUT = 'NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT' Commands = { # name valid states asynchronous 'APPEND': ((AUTH, SELECTED), False), 'AUTHENTICATE': ((NONAUTH,), False), 'CAPABILITY': ((NONAUTH, AUTH, SELECTED), True), 'CHECK': ((SELECTED,), True), 'CLOSE': ((SELECTED,), False), 'COMPRESS': ((AUTH,), False), 'COPY': ((SELECTED,), True), 'CREATE': ((AUTH, SELECTED), True), 'DELETE': ((AUTH, SELECTED), True), 'DELETEACL': ((AUTH, SELECTED), True), 'ENABLE': ((AUTH,), False), 'EXAMINE': ((AUTH, SELECTED), False), 'EXPUNGE': ((SELECTED,), True), 'FETCH': ((SELECTED,), True), 'GETACL': ((AUTH, SELECTED), True), 'GETANNOTATION':((AUTH, SELECTED), True), 'GETQUOTA': ((AUTH, SELECTED), True), 'GETQUOTAROOT': ((AUTH, SELECTED), True), 'ID': ((NONAUTH, AUTH, LOGOUT, SELECTED), True), 'IDLE': ((SELECTED,), False), 'LIST': ((AUTH, SELECTED), True), 'LOGIN': ((NONAUTH,), False), 'LOGOUT': ((NONAUTH, AUTH, LOGOUT, SELECTED), False), 'LSUB': ((AUTH, SELECTED), True), 'MYRIGHTS': ((AUTH, SELECTED), True), 'NAMESPACE': ((AUTH, SELECTED), True), 'NOOP': ((NONAUTH, AUTH, SELECTED), True), 'PARTIAL': ((SELECTED,), True), 'PROXYAUTH': ((AUTH,), False), 'RENAME': ((AUTH, SELECTED), True), 'SEARCH': ((SELECTED,), True), 'SELECT': ((AUTH, SELECTED), False), 'SETACL': ((AUTH, SELECTED), False), 'SETANNOTATION':((AUTH, SELECTED), True), 'SETQUOTA': ((AUTH, SELECTED), False), 'SORT': ((SELECTED,), True), 'STARTTLS': ((NONAUTH,), False), 'STATUS': ((AUTH, SELECTED), True), 'STORE': ((SELECTED,), True), 'SUBSCRIBE': ((AUTH, SELECTED), False), 'THREAD': ((SELECTED,), True), 'UID': ((SELECTED,), True), 'UNSUBSCRIBE': ((AUTH, SELECTED), False), } UID_direct = ('SEARCH', 'SORT', 'THREAD') def Int2AP(num): """string = Int2AP(num) Return 'num' converted to a string using characters from the set 'A'..'P' """ val, a2p = [], 'ABCDEFGHIJKLMNOP' num = int(abs(num)) while num: num, mod = divmod(num, 16) val.insert(0, a2p[mod]) return ''.join(val) class Request(object): """Private class to represent a request awaiting response.""" def __init__(self, parent, name=None, callback=None, cb_arg=None, cb_self=False): self.parent = parent self.name = name self.callback = callback # Function called to process result if not cb_self: self.callback_arg = cb_arg # Optional arg passed to "callback" else: self.callback_arg = (self, cb_arg) # Self reference required in callback arg self.tag = '%s%s' % (parent.tagpre, parent.tagnum) parent.tagnum += 1 self.ready = threading.Event() self.response = None self.aborted = None self.data = None def abort(self, typ, val): self.aborted = (typ, val) self.deliver(None) def get_response(self, exc_fmt=None): self.callback = None if __debug__: self.parent._log(3, '%s:%s.ready.wait' % (self.name, self.tag)) self.ready.wait() if self.aborted is not None: typ, val = self.aborted if exc_fmt is None: exc_fmt = '%s - %%s' % typ raise typ(exc_fmt % str(val)) return self.response def deliver(self, response): if self.callback is not None: self.callback((response, self.callback_arg, self.aborted)) return self.response = response self.ready.set() if __debug__: self.parent._log(3, '%s:%s.ready.set' % (self.name, self.tag)) class IMAP4(object): """Threaded IMAP4 client class. Instantiate with: IMAP4(host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) host - host's name (default: localhost); port - port number (default: standard IMAP4 port); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); timeout - timeout in seconds when expecting a command response (default: no timeout), debug_buf_lvl - debug level at which buffering is turned off. All IMAP4rev1 commands are supported by methods of the same name. Each command returns a tuple: (type, [data, ...]) where 'type' is usually 'OK' or 'NO', and 'data' is either the text from the tagged response, or untagged results from command. Each 'data' is either a string, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). Errors raise the exception class .error(""). IMAP4 server errors raise .abort(""), which is a sub-class of 'error'. Mailbox status changes from READ-WRITE to READ-ONLY raise the exception class .readonly(""), which is a sub-class of 'abort'. "error" exceptions imply a program error. "abort" exceptions imply the connection should be reset, and the command re-tried. "readonly" exceptions imply the command should be re-tried. All commands take two optional named arguments: 'callback' and 'cb_arg' If 'callback' is provided then the command is asynchronous, so after the command is queued for transmission, the call returns immediately with the tuple (None, None). The result will be posted by invoking "callback" with one arg, a tuple: callback((result, cb_arg, None)) or, if there was a problem: callback((None, cb_arg, (exception class, reason))) Otherwise the command is synchronous (waits for result). But note that state-changing commands will both block until previous commands have completed, and block subsequent commands until they have finished. All (non-callback) arguments to commands are converted to strings, except for AUTHENTICATE, and the last argument to APPEND which is passed as an IMAP4 literal. If necessary (the string contains any non-printing characters or white-space and isn't enclosed with either parentheses or double or single quotes) each string is quoted. However, the 'password' argument to the LOGIN command is always quoted. If you want to avoid having an argument string quoted (eg: the 'flags' argument to STORE) then enclose the string in parentheses (eg: "(\Deleted)"). If you are using "sequence sets" containing the wildcard character '*', then enclose the argument in single quotes: the quotes will be removed and the resulting string passed unquoted. Note also that you can pass in an argument with a type that doesn't evaluate to 'string_types' (eg: 'bytearray') and it will be converted to a string without quoting. There is one instance variable, 'state', that is useful for tracking whether the client needs to login to the server. If it has the value "AUTH" after instantiating the class, then the connection is pre-authenticated (otherwise it will be "NONAUTH"). Selecting a mailbox changes the state to be "SELECTED", closing a mailbox changes back to "AUTH", and once the client has logged out, the state changes to "LOGOUT" and no further commands may be issued. Note: to use this module, you must read the RFCs pertaining to the IMAP4 protocol, as the semantics of the arguments to each IMAP4 command are left to the invoker, not to mention the results. Also, most IMAP servers implement a sub-set of the commands available here. Note also that you must call logout() to shut down threads before discarding an instance. """ class error(Exception): pass # Logical errors - debug required class abort(error): pass # Service errors - close and retry class readonly(abort): pass # Mailbox status changed to READ-ONLY # These must be encoded according to utf8 setting in _mode_xxx(): _literal = br'.*{(?P\d+)}$' _untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' continuation_cre = re.compile(r'\+( (?P.*))?') mapCRLF_cre = re.compile(r'\r\n|\r|\n') # Need to quote "atom-specials" :- # "(" / ")" / "{" / SP / 0x00 - 0x1f / 0x7f / "%" / "*" / DQUOTE / "\" / "]" # so match not the inverse set mustquote_cre = re.compile(r"[^!#$&'+,./0-9:;<=>?@A-Z\[^_`a-z|}~-]") response_code_cre = re.compile(r'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') untagged_response_cre = re.compile(r'\* (?P[A-Z-]+)( (?P.*))?') def __init__(self, host=None, port=None, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.state = NONAUTH # IMAP4 protocol state self.literal = None # A literal argument to a command self.tagged_commands = {} # Tagged commands awaiting response self.untagged_responses = [] # [[typ: [data, ...]], ...] self.mailbox = None # Current mailbox selected self.is_readonly = False # READ-ONLY desired state self.idle_rqb = None # Server IDLE Request - see _IdleCont self.idle_timeout = None # Must prod server occasionally self._expecting_data = False # Expecting message data self._expecting_data_len = 0 # How many characters we expect self._accumulated_data = [] # Message data accumulated so far self._literal_expected = None # Message data descriptor self.compressor = None # COMPRESS/DEFLATE if not None self.decompressor = None self._tls_established = False # Create unique tag for this session, # and compile tagged response matcher. self.tagnum = 0 self.tagpre = Int2AP(random.randint(4096, 65535)) self.tagre = re.compile(r'(?P' + self.tagpre + r'\d+) (?P[A-Z]+) ?(?P.*)') self._mode_ascii() # Only option in py2 if __debug__: self._init_debug(debug, debug_file, debug_buf_lvl) self.resp_timeout = timeout # Timeout waiting for command response if timeout is not None and timeout < READ_POLL_TIMEOUT: self.read_poll_timeout = timeout else: self.read_poll_timeout = READ_POLL_TIMEOUT self.read_size = READ_SIZE # Open socket to server. self.open(host, port) if __debug__: if debug: self._mesg('connected to %s on port %s' % (self.host, self.port)) # Threading if identifier is not None: self.identifier = identifier else: self.identifier = self.host if self.identifier: self.identifier += ' ' self.Terminate = self.TerminateReader = False self.state_change_free = threading.Event() self.state_change_pending = threading.Lock() self.commands_lock = threading.Lock() self.idle_lock = threading.Lock() self.ouq = queue.Queue(10) self.inq = queue.Queue() self.wrth = threading.Thread(target=self._writer) self.wrth.setDaemon(True) self.wrth.start() self.rdth = threading.Thread(target=self._reader) self.rdth.setDaemon(True) self.rdth.start() self.inth = threading.Thread(target=self._handler) self.inth.setDaemon(True) self.inth.start() # Get server welcome message, # request and store CAPABILITY response. try: self.welcome = self._request_push(name='welcome', tag='continuation').get_response('IMAP4 protocol error: %s')[1] if self._get_untagged_response('PREAUTH'): self.state = AUTH if __debug__: self._log(1, 'state => AUTH') elif self._get_untagged_response('OK'): if __debug__: self._log(1, 'state => NONAUTH') else: raise self.error('unrecognised server welcome message: %s' % repr(self.welcome)) typ, dat = self.capability() if dat == [None]: raise self.error('no CAPABILITY response from server') self.capabilities = tuple(dat[-1].upper().split()) if __debug__: self._log(1, 'CAPABILITY: %r' % (self.capabilities,)) for version in AllowedVersions: if not version in self.capabilities: continue self.PROTOCOL_VERSION = version break else: raise self.error('server not IMAP4 compliant') except: self._close_threads() raise def __getattr__(self, attr): # Allow UPPERCASE variants of IMAP4 command methods. if attr in Commands: return getattr(self, attr.lower()) raise AttributeError("Unknown IMAP4 command: '%s'" % attr) def _mode_ascii(self): self.utf8_enabled = False self._encoding = 'ascii' if bytes != str: self.literal_cre = re.compile(self._literal, re.ASCII) self.untagged_status_cre = re.compile(self._untagged_status, re.ASCII) else: self.literal_cre = re.compile(self._literal) self.untagged_status_cre = re.compile(self._untagged_status) def _mode_utf8(self): self.utf8_enabled = True self._encoding = 'utf-8' if bytes != str: self.literal_cre = re.compile(self._literal) self.untagged_status_cre = re.compile(self._untagged_status) else: self.literal_cre = re.compile(self._literal, re.UNICODE) self.untagged_status_cre = re.compile(self._untagged_status, re.UNICODE) # Overridable methods def open(self, host=None, port=None): """open(host=None, port=None) Setup connection to remote server on "host:port" (default: localhost:standard IMAP4 port). This connection will be used by the routines: read, send, shutdown, socket.""" self.host = self._choose_nonull_or_dflt('', host) self.port = self._choose_nonull_or_dflt(IMAP4_PORT, port) self.sock = self.open_socket() self.read_fd = self.sock.fileno() def open_socket(self): """open_socket() Open socket choosing first address family available.""" msg = (-1, 'could not open socket') for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: s = socket.socket(af, socktype, proto) except socket.error as m: msg = m continue try: for i in (0, 1): try: s.connect(sa) break except socket.error as m: msg = m if len(msg.args) < 2 or msg.args[0] != errno.EINTR: raise else: raise socket.error(msg) except socket.error as m: msg = m s.close() continue break else: raise socket.error(msg) return s def ssl_wrap_socket(self): try: import ssl TLS_MAP = {} if hasattr(ssl, "PROTOCOL_TLSv1_2"): TLS_MAP[TLS_SECURE] = { "tls1_2": ssl.PROTOCOL_TLSv1_2, "tls1_1": ssl.PROTOCOL_TLSv1_1, } else: TLS_MAP[TLS_SECURE] = {} TLS_MAP[TLS_NO_SSL] = TLS_MAP[TLS_SECURE].copy() TLS_MAP[TLS_NO_SSL].update({ "tls1": ssl.PROTOCOL_TLSv1, }) TLS_MAP[TLS_COMPAT] = TLS_MAP[TLS_NO_SSL].copy() TLS_MAP[TLS_COMPAT].update({ "ssl23": ssl.PROTOCOL_SSLv23, None: ssl.PROTOCOL_SSLv23, }) if hasattr(ssl, "PROTOCOL_SSLv3"): # Might not be available. TLS_MAP[TLS_COMPAT].update({ "ssl3": ssl.PROTOCOL_SSLv3 }) if self.ca_certs is not None: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE if self.tls_level not in TLS_MAP: raise RuntimeError("unknown tls_level: %s" % self.tls_level) if self.ssl_version not in TLS_MAP[self.tls_level]: raise socket.sslerror("Invalid SSL version '%s' requested for tls_version '%s'" % (self.ssl_version, self.tls_level)) ssl_version = TLS_MAP[self.tls_level][self.ssl_version] if getattr(ssl, 'HAS_SNI', False): ctx = ssl.SSLContext(ssl_version) ctx.verify_mode = cert_reqs if self.ca_certs is not None: ctx.load_verify_locations(self.ca_certs) if self.certfile or self.keyfile: ctx.load_cert_chain(self.certfile, self.keyfile) self.sock = ctx.wrap_socket(self.sock, server_hostname=self.host) else: self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, ca_certs=self.ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) ssl_exc = ssl.SSLError self.read_fd = self.sock.fileno() except ImportError: # No ssl module, and socket.ssl has no fileno(), and does not allow certificate verification raise socket.sslerror("imaplib SSL mode does not work without ssl module") if self.cert_verify_cb is not None: cert_err = self.cert_verify_cb(self.sock.getpeercert(), self.host) if cert_err: raise ssl_exc(cert_err) # Allow sending of keep-alive messages - seems to prevent some servers # from closing SSL, leading to deadlocks. self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) def start_compressing(self): """start_compressing() Enable deflate compression on the socket (RFC 4978).""" # rfc 1951 - pure DEFLATE, so use -15 for both windows self.decompressor = zlib.decompressobj(-15) self.compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) def read(self, size): """data = read(size) Read at most 'size' bytes from remote.""" if self.decompressor is None: return self.sock.recv(size) if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: data = self.sock.recv(READ_SIZE) return self.decompressor.decompress(data, size) def send(self, data): """send(data) Send 'data' to remote.""" if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: data = bytes(data, 'ASCII') self.sock.sendall(data) def shutdown(self): """shutdown() Close I/O established in "open".""" try: self.sock.shutdown(socket.SHUT_RDWR) except Exception as e: # The server might already have closed the connection if e.errno != errno.ENOTCONN: raise finally: self.sock.close() def socket(self): """socket = socket() Return socket instance used to connect to IMAP4 server.""" return self.sock # Utility methods def enable_compression(self): """enable_compression() Ask the server to start compressing the connection. Should be called from user of this class after instantiation, as in: if 'COMPRESS=DEFLATE' in imapobj.capabilities: imapobj.enable_compression()""" try: typ, dat = self._simple_command('COMPRESS', 'DEFLATE') if typ == 'OK': self.start_compressing() if __debug__: self._log(1, 'Enabled COMPRESS=DEFLATE') finally: self._release_state_change() def pop_untagged_responses(self): """ for typ,data in pop_untagged_responses(): pass Generator for any remaining untagged responses. Returns and removes untagged responses in order of reception. Use at your own risk!""" while self.untagged_responses: self.commands_lock.acquire() try: yield self.untagged_responses.pop(0) finally: self.commands_lock.release() def recent(self, **kw): """(typ, [data]) = recent() Return 'RECENT' responses if any exist, else prompt server for an update using the 'NOOP' command. 'data' is None if no new messages, else list of RECENT responses, most recent last.""" name = 'RECENT' typ, dat = self._untagged_response(None, [None], name) if dat != [None]: return self._deliver_dat(typ, dat, kw) kw['untagged_response'] = name return self.noop(**kw) # Prod server for response def response(self, code, **kw): """(code, [data]) = response(code) Return data for response 'code' if received, or None. Old value for response 'code' is cleared.""" typ, dat = self._untagged_response(code, [None], code.upper()) return self._deliver_dat(typ, dat, kw) # IMAP4 commands def append(self, mailbox, flags, date_time, message, **kw): """(typ, [data]) = append(mailbox, flags, date_time, message) Append message to named mailbox. All args except `message' can be None.""" name = 'APPEND' if not mailbox: mailbox = 'INBOX' if flags: if (flags[0],flags[-1]) != ('(',')'): flags = '(%s)' % flags else: flags = None if date_time: date_time = Time2Internaldate(date_time) else: date_time = None literal = self.mapCRLF_cre.sub(CRLF, message) if self.utf8_enabled: literal = b'UTF8 (' + literal + b')' self.literal = literal try: return self._simple_command(name, mailbox, flags, date_time, **kw) finally: self._release_state_change() def authenticate(self, mechanism, authobject, **kw): """(typ, [data]) = authenticate(mechanism, authobject) Authenticate command - requires response processing. 'mechanism' specifies which authentication mechanism is to be used - it must appear in .capabilities in the form AUTH=. 'authobject' must be a callable object: data = authobject(response) It will be called to process server continuation responses. It should return data that will be encoded and sent to server. It should return None if the client abort response '*' should be sent instead.""" self.literal = _Authenticator(authobject).process try: typ, dat = self._simple_command('AUTHENTICATE', mechanism.upper()) if typ != 'OK': self._deliver_exc(self.error, dat[-1], kw) self.state = AUTH if __debug__: self._log(1, 'state => AUTH') finally: self._release_state_change() return self._deliver_dat(typ, dat, kw) def capability(self, **kw): """(typ, [data]) = capability() Fetch capabilities list from server.""" name = 'CAPABILITY' kw['untagged_response'] = name return self._simple_command(name, **kw) def check(self, **kw): """(typ, [data]) = check() Checkpoint mailbox on server.""" return self._simple_command('CHECK', **kw) def close(self, **kw): """(typ, [data]) = close() Close currently selected mailbox. Deleted messages are removed from writable mailbox. This is the recommended command before 'LOGOUT'.""" if self.state != 'SELECTED': raise self.error('No mailbox selected.') try: typ, dat = self._simple_command('CLOSE') finally: self.state = AUTH if __debug__: self._log(1, 'state => AUTH') self._release_state_change() return self._deliver_dat(typ, dat, kw) def copy(self, message_set, new_mailbox, **kw): """(typ, [data]) = copy(message_set, new_mailbox) Copy 'message_set' messages onto end of 'new_mailbox'.""" return self._simple_command('COPY', message_set, new_mailbox, **kw) def create(self, mailbox, **kw): """(typ, [data]) = create(mailbox) Create new mailbox.""" return self._simple_command('CREATE', mailbox, **kw) def delete(self, mailbox, **kw): """(typ, [data]) = delete(mailbox) Delete old mailbox.""" return self._simple_command('DELETE', mailbox, **kw) def deleteacl(self, mailbox, who, **kw): """(typ, [data]) = deleteacl(mailbox, who) Delete the ACLs (remove any rights) set for who on mailbox.""" return self._simple_command('DELETEACL', mailbox, who, **kw) def enable(self, capability): """Send an RFC5161 enable string to the server. (typ, [data]) = .enable(capability) """ if 'ENABLE' not in self.capabilities: raise self.error("Server does not support ENABLE") typ, data = self._simple_command('ENABLE', capability) if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): self._mode_utf8() return typ, data def examine(self, mailbox='INBOX', **kw): """(typ, [data]) = examine(mailbox='INBOX') Select a mailbox for READ-ONLY access. (Flushes all untagged responses.) 'data' is count of messages in mailbox ('EXISTS' response). Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so other responses should be obtained via "response('FLAGS')" etc.""" return self.select(mailbox=mailbox, readonly=True, **kw) def expunge(self, **kw): """(typ, [data]) = expunge() Permanently remove deleted items from selected mailbox. Generates 'EXPUNGE' response for each deleted message. 'data' is list of 'EXPUNGE'd message numbers in order received.""" name = 'EXPUNGE' kw['untagged_response'] = name return self._simple_command(name, **kw) def fetch(self, message_set, message_parts, **kw): """(typ, [data, ...]) = fetch(message_set, message_parts) Fetch (parts of) messages. 'message_parts' should be a string of selected parts enclosed in parentheses, eg: "(UID BODY[TEXT])". 'data' are tuples of message part envelope and data, followed by a string containing the trailer.""" name = 'FETCH' kw['untagged_response'] = name return self._simple_command(name, message_set, message_parts, **kw) def getacl(self, mailbox, **kw): """(typ, [data]) = getacl(mailbox) Get the ACLs for a mailbox.""" kw['untagged_response'] = 'ACL' return self._simple_command('GETACL', mailbox, **kw) def getannotation(self, mailbox, entry, attribute, **kw): """(typ, [data]) = getannotation(mailbox, entry, attribute) Retrieve ANNOTATIONs.""" kw['untagged_response'] = 'ANNOTATION' return self._simple_command('GETANNOTATION', mailbox, entry, attribute, **kw) def getquota(self, root, **kw): """(typ, [data]) = getquota(root) Get the quota root's resource usage and limits. (Part of the IMAP4 QUOTA extension defined in rfc2087.)""" kw['untagged_response'] = 'QUOTA' return self._simple_command('GETQUOTA', root, **kw) def getquotaroot(self, mailbox, **kw): # Hmmm, this is non-std! Left for backwards-compatibility, sigh. # NB: usage should have been defined as: # (typ, [QUOTAROOT responses...]) = getquotaroot(mailbox) # (typ, [QUOTA responses...]) = response('QUOTA') """(typ, [[QUOTAROOT responses...], [QUOTA responses...]]) = getquotaroot(mailbox) Get the list of quota roots for the named mailbox.""" typ, dat = self._simple_command('GETQUOTAROOT', mailbox) typ, quota = self._untagged_response(typ, dat, 'QUOTA') typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') return self._deliver_dat(typ, [quotaroot, quota], kw) def id(self, *kv_pairs, **kw): """(typ, [data]) = .id(kv_pairs) 'kv_pairs' is a possibly empty list of keys and values. 'data' is a list of ID key value pairs or NIL. NB: a single argument is assumed to be correctly formatted and is passed through unchanged (for backward compatibility with earlier version). Exchange information for problem analysis and determination. The ID extension is defined in RFC 2971. """ name = 'ID' kw['untagged_response'] = name if not kv_pairs: data = 'NIL' elif len(kv_pairs) == 1: data = kv_pairs[0] # Assume invoker passing correctly formatted string (back-compat) else: data = '(%s)' % ' '.join([(arg and self._quote(arg) or 'NIL') for arg in kv_pairs]) return self._simple_command(name, data, **kw) def idle(self, timeout=None, **kw): """"(typ, [data]) = idle(timeout=None) Put server into IDLE mode until server notifies some change, or 'timeout' (secs) occurs (default: 29 minutes), or another IMAP4 command is scheduled.""" name = 'IDLE' self.literal = _IdleCont(self, timeout).process try: return self._simple_command(name, **kw) finally: self._release_state_change() def list(self, directory='""', pattern='*', **kw): """(typ, [data]) = list(directory='""', pattern='*') List mailbox names in directory matching pattern. 'data' is list of LIST responses. NB: for 'pattern': % matches all except separator ( so LIST "" "%" returns names at root) * matches all (so LIST "" "*" returns whole directory tree from root)""" name = 'LIST' kw['untagged_response'] = name return self._simple_command(name, directory, pattern, **kw) def login(self, user, password, **kw): """(typ, [data]) = login(user, password) Identify client using plaintext password. NB: 'password' will be quoted.""" try: typ, dat = self._simple_command('LOGIN', user, self._quote(password)) if typ != 'OK': self._deliver_exc(self.error, dat[-1], kw) self.state = AUTH if __debug__: self._log(1, 'state => AUTH') finally: self._release_state_change() return self._deliver_dat(typ, dat, kw) def login_cram_md5(self, user, password, **kw): """(typ, [data]) = login_cram_md5(user, password) Force use of CRAM-MD5 authentication.""" self.user, self.password = user, password return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH, **kw) def _CRAM_MD5_AUTH(self, challenge): """Authobject to use with CRAM-MD5 authentication.""" import hmac pwd = (self.password.encode('ASCII') if isinstance(self.password, str) else self.password) return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() def logout(self, **kw): """(typ, [data]) = logout() Shutdown connection to server. Returns server 'BYE' response. NB: You must call this to shut down threads before discarding an instance.""" self.state = LOGOUT if __debug__: self._log(1, 'state => LOGOUT') try: try: typ, dat = self._simple_command('LOGOUT') except: typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] if __debug__: self._log(1, dat) self._close_threads() finally: self._release_state_change() if __debug__: self._log(1, 'connection closed') bye = self._get_untagged_response('BYE', leave=True) if bye: typ, dat = 'BYE', bye return self._deliver_dat(typ, dat, kw) def lsub(self, directory='""', pattern='*', **kw): """(typ, [data, ...]) = lsub(directory='""', pattern='*') List 'subscribed' mailbox names in directory matching pattern. 'data' are tuples of message part envelope and data.""" name = 'LSUB' kw['untagged_response'] = name return self._simple_command(name, directory, pattern, **kw) def myrights(self, mailbox, **kw): """(typ, [data]) = myrights(mailbox) Show my ACLs for a mailbox (i.e. the rights that I have on mailbox).""" name = 'MYRIGHTS' kw['untagged_response'] = name return self._simple_command(name, mailbox, **kw) def namespace(self, **kw): """(typ, [data, ...]) = namespace() Returns IMAP namespaces ala rfc2342.""" name = 'NAMESPACE' kw['untagged_response'] = name return self._simple_command(name, **kw) def noop(self, **kw): """(typ, [data]) = noop() Send NOOP command.""" if __debug__: self._dump_ur(3) return self._simple_command('NOOP', **kw) def partial(self, message_num, message_part, start, length, **kw): """(typ, [data, ...]) = partial(message_num, message_part, start, length) Fetch truncated part of a message. 'data' is tuple of message part envelope and data. NB: obsolete.""" name = 'PARTIAL' kw['untagged_response'] = 'FETCH' return self._simple_command(name, message_num, message_part, start, length, **kw) def proxyauth(self, user, **kw): """(typ, [data]) = proxyauth(user) Assume authentication as 'user'. (Allows an authorised administrator to proxy into any user's mailbox.)""" try: return self._simple_command('PROXYAUTH', user, **kw) finally: self._release_state_change() def rename(self, oldmailbox, newmailbox, **kw): """(typ, [data]) = rename(oldmailbox, newmailbox) Rename old mailbox name to new.""" return self._simple_command('RENAME', oldmailbox, newmailbox, **kw) def search(self, charset, *criteria, **kw): """(typ, [data]) = search(charset, criterion, ...) Search mailbox for matching messages. If UTF8 is enabled, charset MUST be None. 'data' is space separated list of matching message numbers.""" name = 'SEARCH' kw['untagged_response'] = name if charset: if self.utf8_enabled: raise self.error("Non-None charset not valid in UTF8 mode") return self._simple_command(name, 'CHARSET', charset, *criteria, **kw) return self._simple_command(name, *criteria, **kw) def select(self, mailbox='INBOX', readonly=False, **kw): """(typ, [data]) = select(mailbox='INBOX', readonly=False) Select a mailbox. (Flushes all untagged responses.) 'data' is count of messages in mailbox ('EXISTS' response). Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so other responses should be obtained via "response('FLAGS')" etc.""" self.mailbox = mailbox self.is_readonly = bool(readonly) if readonly: name = 'EXAMINE' else: name = 'SELECT' try: rqb = self._command(name, mailbox) typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) if typ != 'OK': if self.state == SELECTED: self.state = AUTH if __debug__: self._log(1, 'state => AUTH') if typ == 'BAD': self._deliver_exc(self.error, '%s command error: %s %s. Data: %.100s' % (name, typ, dat, mailbox), kw) return self._deliver_dat(typ, dat, kw) self.state = SELECTED if __debug__: self._log(1, 'state => SELECTED') finally: self._release_state_change() if self._get_untagged_response('READ-ONLY', leave=True) and not readonly: if __debug__: self._dump_ur(1) self._deliver_exc(self.readonly, '%s is not writable' % mailbox, kw) typ, dat = self._untagged_response(typ, [None], 'EXISTS') return self._deliver_dat(typ, dat, kw) def setacl(self, mailbox, who, what, **kw): """(typ, [data]) = setacl(mailbox, who, what) Set a mailbox acl.""" try: return self._simple_command('SETACL', mailbox, who, what, **kw) finally: self._release_state_change() def setannotation(self, *args, **kw): """(typ, [data]) = setannotation(mailbox[, entry, attribute]+) Set ANNOTATIONs.""" kw['untagged_response'] = 'ANNOTATION' return self._simple_command('SETANNOTATION', *args, **kw) def setquota(self, root, limits, **kw): """(typ, [data]) = setquota(root, limits) Set the quota root's resource limits.""" kw['untagged_response'] = 'QUOTA' try: return self._simple_command('SETQUOTA', root, limits, **kw) finally: self._release_state_change() def sort(self, sort_criteria, charset, *search_criteria, **kw): """(typ, [data]) = sort(sort_criteria, charset, search_criteria, ...) IMAP4rev1 extension SORT command.""" name = 'SORT' if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): sort_criteria = '(%s)' % sort_criteria kw['untagged_response'] = name return self._simple_command(name, sort_criteria, charset, *search_criteria, **kw) def starttls(self, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level=TLS_COMPAT, **kw): """(typ, [data]) = starttls(keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", tls_level="tls_compat") Start TLS negotiation as per RFC 2595.""" name = 'STARTTLS' if name not in self.capabilities: raise self.abort('TLS not supported by server') if self._tls_established: raise self.abort('TLS session already established') # Must now shutdown reader thread after next response, and restart after changing read_fd self.read_size = 1 # Don't consume TLS handshake self.TerminateReader = True try: typ, dat = self._simple_command(name) finally: self._release_state_change() self.rdth.join() self.TerminateReader = False self.read_size = READ_SIZE if typ != 'OK': # Restart reader thread and error self.rdth = threading.Thread(target=self._reader) self.rdth.setDaemon(True) self.rdth.start() raise self.error("Couldn't establish TLS session: %s" % dat) self.keyfile = keyfile self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version self.tls_level = tls_level try: self.ssl_wrap_socket() finally: # Restart reader thread self.rdth = threading.Thread(target=self._reader) self.rdth.setDaemon(True) self.rdth.start() typ, dat = self.capability() if dat == [None]: raise self.error('no CAPABILITY response from server') self.capabilities = tuple(dat[-1].upper().split()) self._tls_established = True typ, dat = self._untagged_response(typ, dat, name) return self._deliver_dat(typ, dat, kw) def status(self, mailbox, names, **kw): """(typ, [data]) = status(mailbox, names) Request named status conditions for mailbox.""" name = 'STATUS' kw['untagged_response'] = name return self._simple_command(name, mailbox, names, **kw) def store(self, message_set, command, flags, **kw): """(typ, [data]) = store(message_set, command, flags) Alters flag dispositions for messages in mailbox.""" if (flags[0],flags[-1]) != ('(',')'): flags = '(%s)' % flags # Avoid quoting the flags kw['untagged_response'] = 'FETCH' return self._simple_command('STORE', message_set, command, flags, **kw) def subscribe(self, mailbox, **kw): """(typ, [data]) = subscribe(mailbox) Subscribe to new mailbox.""" try: return self._simple_command('SUBSCRIBE', mailbox, **kw) finally: self._release_state_change() def thread(self, threading_algorithm, charset, *search_criteria, **kw): """(type, [data]) = thread(threading_alogrithm, charset, search_criteria, ...) IMAPrev1 extension THREAD command.""" name = 'THREAD' kw['untagged_response'] = name return self._simple_command(name, threading_algorithm, charset, *search_criteria, **kw) def uid(self, command, *args, **kw): """(typ, [data]) = uid(command, arg, ...) Execute "command arg ..." with messages identified by UID, rather than message number. Assumes 'command' is legal in current state. Returns response appropriate to 'command'.""" command = command.upper() if command in UID_direct: resp = command else: resp = 'FETCH' kw['untagged_response'] = resp return self._simple_command('UID', command, *args, **kw) def unsubscribe(self, mailbox, **kw): """(typ, [data]) = unsubscribe(mailbox) Unsubscribe from old mailbox.""" try: return self._simple_command('UNSUBSCRIBE', mailbox, **kw) finally: self._release_state_change() def xatom(self, name, *args, **kw): """(typ, [data]) = xatom(name, arg, ...) Allow simple extension commands notified by server in CAPABILITY response. Assumes extension command 'name' is legal in current state. Returns response appropriate to extension command 'name'.""" name = name.upper() if not name in Commands: Commands[name] = ((self.state,), False) try: return self._simple_command(name, *args, **kw) finally: self._release_state_change() # Internal methods def _append_untagged(self, typ, dat): # Append new 'dat' to end of last untagged response if same 'typ', # else append new response. if dat is None: dat = '' self.commands_lock.acquire() if self.untagged_responses: urn, urd = self.untagged_responses[-1] if urn != typ: urd = None else: urd = None if urd is None: urd = [] self.untagged_responses.append([typ, urd]) urd.append(dat) self.commands_lock.release() if __debug__: self._log(5, 'untagged_responses[%s] %s += ["%.80s"]' % (typ, len(urd)-1, dat)) def _check_bye(self): bye = self._get_untagged_response('BYE', leave=True) if bye: if str != bytes: raise self.abort(bye[-1].decode('ASCII', 'replace')) else: raise self.abort(bye[-1]) def _checkquote(self, arg): # Must quote command args if "atom-specials" present, # and not already quoted. NB: single quotes are removed. if not isinstance(arg, string_types): return arg if len(arg) >= 2 and (arg[0],arg[-1]) in (('(',')'),('"','"')): return arg if len(arg) >= 2 and (arg[0],arg[-1]) in (("'","'"),): return arg[1:-1] if arg and self.mustquote_cre.search(arg) is None: return arg return self._quote(arg) def _choose_nonull_or_dflt(self, dflt, *args): if isinstance(dflt, string_types): dflttyp = string_types # Allow any string type else: dflttyp = type(dflt) for arg in args: if arg is not None: if isinstance(arg, dflttyp): return arg if __debug__: self._log(0, 'bad arg is %s, expecting %s' % (type(arg), dflttyp)) return dflt def _command(self, name, *args, **kw): if Commands[name][CMD_VAL_ASYNC]: cmdtyp = 'async' else: cmdtyp = 'sync' if __debug__: self._log(1, '[%s] %s %s' % (cmdtyp, name, args)) if __debug__: self._log(3, 'state_change_pending.acquire') self.state_change_pending.acquire() self._end_idle() if cmdtyp == 'async': self.state_change_pending.release() if __debug__: self._log(3, 'state_change_pending.release') else: # Need to wait for all async commands to complete self._check_bye() self.commands_lock.acquire() if self.tagged_commands: self.state_change_free.clear() need_event = True else: need_event = False self.commands_lock.release() if need_event: if __debug__: self._log(3, 'sync command %s waiting for empty commands Q' % name) self.state_change_free.wait() if __debug__: self._log(3, 'sync command %s proceeding' % name) if self.state not in Commands[name][CMD_VAL_STATES]: self.literal = None raise self.error('command %s illegal in state %s, only allowed in states %s' % (name, self.state, ', '.join(Commands[name][CMD_VAL_STATES]))) self._check_bye() if name in ('EXAMINE', 'SELECT'): self.commands_lock.acquire() self.untagged_responses = [] # Flush all untagged responses self.commands_lock.release() else: for typ in ('OK', 'NO', 'BAD'): while self._get_untagged_response(typ): continue if not self.is_readonly and self._get_untagged_response('READ-ONLY', leave=True): self.literal = None raise self.readonly('mailbox status changed to READ-ONLY') if self.Terminate: raise self.abort('connection closed') rqb = self._request_push(name=name, **kw) data = '%s %s' % (rqb.tag, name) for arg in args: if arg is None: continue data = '%s %s' % (data, self._checkquote(arg)) literal = self.literal if literal is not None: self.literal = None if isinstance(literal, string_types): literator = None data = '%s {%s}' % (data, len(literal)) else: literator = literal if __debug__: self._log(4, 'data=%s' % data) rqb.data = '%s%s' % (data, CRLF) if literal is None: self.ouq.put(rqb) return rqb # Must setup continuation expectancy *before* ouq.put crqb = self._request_push(name=name, tag='continuation') self.ouq.put(rqb) while True: # Wait for continuation response ok, data = crqb.get_response('command: %s => %%s' % name) if __debug__: self._log(4, 'continuation => %s, %s' % (ok, data)) # NO/BAD response? if not ok: break if data == 'go ahead': # Apparently not uncommon broken IMAP4 server response to AUTHENTICATE command data = '' # Send literal if literator is not None: literal = literator(data, rqb) if literal is None: break if literator is not None: # Need new request for next continuation response crqb = self._request_push(name=name, tag='continuation') if __debug__: self._log(4, 'write literal size %s' % len(literal)) crqb.data = '%s%s' % (literal, CRLF) self.ouq.put(crqb) if literator is None: break return rqb def _command_complete(self, rqb, kw): # Called for non-callback commands self._check_bye() typ, dat = rqb.get_response('command: %s => %%s' % rqb.name) if typ == 'BAD': if __debug__: self._print_log() raise self.error('%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) if 'untagged_response' in kw: return self._untagged_response(typ, dat, kw['untagged_response']) return typ, dat def _command_completer(self, cb_arg_list): # Called for callback commands response, cb_arg, error = cb_arg_list rqb, kw = cb_arg rqb.callback = kw['callback'] rqb.callback_arg = kw.get('cb_arg') if error is not None: if __debug__: self._print_log() typ, val = error rqb.abort(typ, val) return bye = self._get_untagged_response('BYE', leave=True) if bye: if str != bytes: rqb.abort(self.abort, bye[-1].decode('ASCII', 'replace')) else: rqb.abort(self.abort, bye[-1]) return typ, dat = response if typ == 'BAD': if __debug__: self._print_log() rqb.abort(self.error, '%s command error: %s %s. Data: %.100s' % (rqb.name, typ, dat, rqb.data)) return if __debug__: self._log(4, '_command_completer(%s, %s, None) = %s' % (response, cb_arg, rqb.tag)) if 'untagged_response' in kw: response = self._untagged_response(typ, dat, kw['untagged_response']) rqb.deliver(response) def _deliver_dat(self, typ, dat, kw): if 'callback' in kw: kw['callback'](((typ, dat), kw.get('cb_arg'), None)) return typ, dat def _deliver_exc(self, exc, dat, kw): if 'callback' in kw: kw['callback']((None, kw.get('cb_arg'), (exc, dat))) raise exc(dat) def _end_idle(self): self.idle_lock.acquire() irqb = self.idle_rqb if irqb is None: self.idle_lock.release() return self.idle_rqb = None self.idle_timeout = None self.idle_lock.release() irqb.data = 'DONE%s' % CRLF self.ouq.put(irqb) if __debug__: self._log(2, 'server IDLE finished') def _get_untagged_response(self, name, leave=False): self.commands_lock.acquire() for i, (typ, dat) in enumerate(self.untagged_responses): if typ == name: if not leave: del self.untagged_responses[i] self.commands_lock.release() if __debug__: self._log(5, '_get_untagged_response(%s) => %.80s' % (name, dat)) return dat self.commands_lock.release() return None def _match(self, cre, s): # Run compiled regular expression 'cre' match method on 's'. # Save result, return success. self.mo = cre.match(s) return self.mo is not None def _put_response(self, resp): if self._expecting_data: rlen = len(resp) dlen = min(self._expecting_data_len, rlen) if __debug__: self._log(5, '_put_response expecting data len %s, got %s' % (self._expecting_data_len, rlen)) self._expecting_data_len -= dlen self._expecting_data = (self._expecting_data_len != 0) if rlen <= dlen: self._accumulated_data.append(resp) return self._accumulated_data.append(resp[:dlen]) resp = resp[dlen:] if self._accumulated_data: typ, dat = self._literal_expected self._append_untagged(typ, (dat, ''.join(self._accumulated_data))) self._accumulated_data = [] # Protocol mandates all lines terminated by CRLF resp = resp[:-2] if __debug__: self._log(5, '_put_response(%s)' % resp) if 'continuation' in self.tagged_commands: continuation_expected = True else: continuation_expected = False if self._literal_expected is not None: dat = resp if self._match(self.literal_cre, dat): self._literal_expected[1] = dat self._expecting_data = True self._expecting_data_len = int(self.mo.group('size')) if __debug__: self._log(4, 'expecting literal size %s' % self._expecting_data_len) return typ = self._literal_expected[0] self._literal_expected = None if dat: self._append_untagged(typ, dat) # Tail if __debug__: self._log(4, 'literal completed') else: # Command completion response? if self._match(self.tagre, resp): tag = self.mo.group('tag') typ = self.mo.group('type') dat = self.mo.group('data') if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): self._append_untagged(self.mo.group('type'), self.mo.group('data')) if not tag in self.tagged_commands: if __debug__: self._log(1, 'unexpected tagged response: %s' % resp) else: self._request_pop(tag, (typ, [dat])) else: dat2 = None # '*' (untagged) responses? if not self._match(self.untagged_response_cre, resp): if self._match(self.untagged_status_cre, resp): dat2 = self.mo.group('data2') if self.mo is None: # Only other possibility is '+' (continuation) response... if self._match(self.continuation_cre, resp): if not continuation_expected: if __debug__: self._log(1, "unexpected continuation response: '%s'" % resp) return self._request_pop('continuation', (True, self.mo.group('data'))) return if __debug__: self._log(1, "unexpected response: '%s'" % resp) return typ = self.mo.group('type') dat = self.mo.group('data') if dat is None: dat = '' # Null untagged response if dat2: dat = dat + ' ' + dat2 # Is there a literal to come? if self._match(self.literal_cre, dat): self._expecting_data = True self._expecting_data_len = int(self.mo.group('size')) if __debug__: self._log(4, 'read literal size %s' % self._expecting_data_len) self._literal_expected = [typ, dat] return self._append_untagged(typ, dat) if typ in ('OK', 'NO', 'BAD') and self._match(self.response_code_cre, dat): self._append_untagged(self.mo.group('type'), self.mo.group('data')) if typ != 'OK': # NO, BYE, IDLE self._end_idle() # Command waiting for aborted continuation response? if continuation_expected: self._request_pop('continuation', (False, resp)) # Bad news? if typ in ('NO', 'BAD', 'BYE'): if typ == 'BYE': self.Terminate = True if __debug__: self._log(1, '%s response: %s' % (typ, dat)) def _quote(self, arg): return '"%s"' % arg.replace('\\', '\\\\').replace('"', '\\"') def _release_state_change(self): if self.state_change_pending.locked(): self.state_change_pending.release() if __debug__: self._log(3, 'state_change_pending.release') def _request_pop(self, name, data): self.commands_lock.acquire() rqb = self.tagged_commands.pop(name) if not self.tagged_commands: need_event = True else: need_event = False self.commands_lock.release() if __debug__: self._log(4, '_request_pop(%s, %s) [%d] = %s' % (name, data, len(self.tagged_commands), rqb.tag)) rqb.deliver(data) if need_event: if __debug__: self._log(3, 'state_change_free.set') self.state_change_free.set() def _request_push(self, tag=None, name=None, **kw): self.commands_lock.acquire() rqb = Request(self, name=name, **kw) if tag is None: tag = rqb.tag self.tagged_commands[tag] = rqb self.commands_lock.release() if __debug__: self._log(4, '_request_push(%s, %s, %s) = %s' % (tag, name, repr(kw), rqb.tag)) return rqb def _simple_command(self, name, *args, **kw): if 'callback' in kw: # Note: old calling sequence for back-compat with python <2.6 self._command(name, callback=self._command_completer, cb_arg=kw, cb_self=True, *args) return (None, None) return self._command_complete(self._command(name, *args), kw) def _untagged_response(self, typ, dat, name): if typ == 'NO': return typ, dat data = self._get_untagged_response(name) if not data: return typ, [None] while True: dat = self._get_untagged_response(name) if not dat: break data += dat if __debug__: self._log(4, '_untagged_response(%s, ?, %s) => %.80s' % (typ, name, data)) return typ, data # Threads def _close_threads(self): if __debug__: self._log(1, '_close_threads') self.ouq.put(None) self.wrth.join() if __debug__: self._log(1, 'call shutdown') self.shutdown() self.rdth.join() self.inth.join() def _handler(self): resp_timeout = self.resp_timeout threading.currentThread().setName(self.identifier + 'handler') time.sleep(0.1) # Don't start handling before main thread ready if __debug__: self._log(1, 'starting') typ, val = self.abort, 'connection terminated' while not self.Terminate: self.idle_lock.acquire() if self.idle_timeout is not None: timeout = self.idle_timeout - time.time() if timeout <= 0: timeout = 1 if __debug__: if self.idle_rqb is not None: self._log(5, 'server IDLING, timeout=%.2f' % timeout) else: timeout = resp_timeout self.idle_lock.release() try: line = self.inq.get(True, timeout) except queue.Empty: if self.idle_rqb is None: if resp_timeout is not None and self.tagged_commands: if __debug__: self._log(1, 'response timeout') typ, val = self.abort, 'no response after %s secs' % resp_timeout break continue if self.idle_timeout > time.time(): continue if __debug__: self._log(2, 'server IDLE timedout') line = IDLE_TIMEOUT_RESPONSE if line is None: if __debug__: self._log(1, 'inq None - terminating') break if not isinstance(line, string_types): typ, val = line break try: self._put_response(line) except: typ, val = self.error, 'program error: %s - %s' % sys.exc_info()[:2] break self.Terminate = True if __debug__: self._log(1, 'terminating: %s' % repr(val)) while not self.ouq.empty(): try: qel = self.ouq.get_nowait() if qel is not None: qel.abort(typ, val) except queue.Empty: break self.ouq.put(None) self.commands_lock.acquire() for name in list(self.tagged_commands.keys()): rqb = self.tagged_commands.pop(name) rqb.abort(typ, val) self.state_change_free.set() self.commands_lock.release() if __debug__: self._log(3, 'state_change_free.set') if __debug__: self._log(1, 'finished') if hasattr(select_module, "poll"): def _reader(self): threading.currentThread().setName(self.identifier + 'reader') if __debug__: self._log(1, 'starting using poll') def poll_error(state): PollErrors = { select.POLLERR: 'Error', select.POLLHUP: 'Hang up', select.POLLNVAL: 'Invalid request: descriptor not open', } return ' '.join([PollErrors[s] for s in PollErrors.keys() if (s & state)]) if bytes != str: line_part = b'' else: line_part = '' poll = select.poll() poll.register(self.read_fd, select.POLLIN) rxzero = 0 terminate = False read_poll_timeout = self.read_poll_timeout * 1000 # poll() timeout is in millisecs while not (terminate or self.Terminate): if self.state == LOGOUT: timeout = 10 else: timeout = read_poll_timeout try: r = poll.poll(timeout) if __debug__: self._log(5, 'poll => %s' % repr(r)) if not r: continue # Timeout fd,state = r[0] if state & select.POLLIN: data = self.read(self.read_size) # Drain ssl buffer if present start = 0 dlen = len(data) if __debug__: self._log(5, 'rcvd %s' % dlen) if dlen == 0: rxzero += 1 if rxzero > 5: raise IOError("Too many read 0") time.sleep(0.1) continue # Try again rxzero = 0 while True: if bytes != str: stop = data.find(b'\n', start) if stop < 0: line_part += data[start:] break stop += 1 line_part, start, line = \ b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: line_part += data[start:] break stop += 1 line_part, start, line = \ '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) if self.TerminateReader: terminate = True if state & ~(select.POLLIN): raise IOError(poll_error(state)) except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] if __debug__: if not self.Terminate: self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) self.inq.put((self.abort, reason)) break poll.unregister(self.read_fd) if __debug__: self._log(1, 'finished') else: # No "poll" - use select() def _reader(self): threading.currentThread().setName(self.identifier + 'reader') if __debug__: self._log(1, 'starting using select') if bytes != str: line_part = b'' else: line_part = '' rxzero = 0 terminate = False while not (terminate or self.Terminate): if self.state == LOGOUT: timeout = 1 else: timeout = self.read_poll_timeout try: r,w,e = select.select([self.read_fd], [], [], timeout) if __debug__: self._log(5, 'select => %s, %s, %s' % (r,w,e)) if not r: # Timeout continue data = self.read(self.read_size) # Drain ssl buffer if present start = 0 dlen = len(data) if __debug__: self._log(5, 'rcvd %s' % dlen) if dlen == 0: rxzero += 1 if rxzero > 5: raise IOError("Too many read 0") time.sleep(0.1) continue # Try again rxzero = 0 while True: if bytes != str: stop = data.find(b'\n', start) if stop < 0: line_part += data[start:] break stop += 1 line_part, start, line = \ b'', stop, (line_part + data[start:stop]).decode(errors='ignore') else: stop = data.find('\n', start) if stop < 0: line_part += data[start:] break stop += 1 line_part, start, line = \ '', stop, line_part + data[start:stop] if __debug__: self._log(4, '< %s' % line) self.inq.put(line) if self.TerminateReader: terminate = True except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] if __debug__: if not self.Terminate: self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) self.inq.put((self.abort, reason)) break if __debug__: self._log(1, 'finished') def _writer(self): threading.currentThread().setName(self.identifier + 'writer') if __debug__: self._log(1, 'starting') reason = 'Terminated' while not self.Terminate: rqb = self.ouq.get() if rqb is None: break # Outq flushed try: self.send(rqb.data) if __debug__: self._log(4, '> %s' % rqb.data) except: reason = 'socket error: %s - %s' % sys.exc_info()[:2] if __debug__: if not self.Terminate: self._print_log() if self.debug: self.debug += 4 # Output all self._log(1, reason) rqb.abort(self.abort, reason) break self.inq.put((self.abort, reason)) if __debug__: self._log(1, 'finished') # Debugging if __debug__: def _init_debug(self, debug=None, debug_file=None, debug_buf_lvl=None): self.debug_lock = threading.Lock() self.debug = self._choose_nonull_or_dflt(0, debug, Debug) self.debug_file = self._choose_nonull_or_dflt(sys.stderr, debug_file) self.debug_buf_lvl = self._choose_nonull_or_dflt(DFLT_DEBUG_BUF_LVL, debug_buf_lvl) self._cmd_log_len = 20 self._cmd_log_idx = 0 self._cmd_log = {} # Last `_cmd_log_len' interactions if self.debug: self._mesg('imaplib2 version %s' % __version__) self._mesg('imaplib2 debug level %s, buffer level %s' % (self.debug, self.debug_buf_lvl)) def _dump_ur(self, lvl): if lvl > self.debug: return l = self.untagged_responses if not l: return t = '\n\t\t' l = ['%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or '') for x in l] self.debug_lock.acquire() self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) self.debug_lock.release() def _log(self, lvl, line): if lvl > self.debug: return if line[-2:] == CRLF: line = line[:-2] + '\\r\\n' tn = threading.currentThread().getName() if lvl <= 1 or self.debug > self.debug_buf_lvl: self.debug_lock.acquire() self._mesg(line, tn) self.debug_lock.release() if lvl != 1: return # Keep log of last `_cmd_log_len' interactions for debugging. self.debug_lock.acquire() self._cmd_log[self._cmd_log_idx] = (line, tn, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: self._cmd_log_idx = 0 self.debug_lock.release() def _mesg(self, s, tn=None, secs=None): if secs is None: secs = time.time() if tn is None: tn = threading.currentThread().getName() tm = time.strftime('%M:%S', time.localtime(secs)) try: self.debug_file.write(' %s.%02d %s %s\n' % (tm, (secs*100)%100, tn, s)) self.debug_file.flush() finally: pass def _print_log(self): self.debug_lock.acquire() i, n = self._cmd_log_idx, self._cmd_log_len if n: self._mesg('last %d log messages:' % n) while n: try: self._mesg(*self._cmd_log[i]) except: pass i += 1 if i >= self._cmd_log_len: i = 0 n -= 1 self.debug_lock.release() class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: IMAP4_SSL(host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level="tls_compat") host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); keyfile - PEM formatted file that contains your private key (default: None); certfile - PEM formatted certificate chain file (default: None); ca_certs - PEM formatted certificate chain file used to validate server certificates (default: None); cert_verify_cb - function to verify authenticity of server certificates (default: None); ssl_version - SSL version to use (default: "ssl23", choose from: "tls1","ssl3","ssl23"); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); timeout - timeout in seconds when expecting a command response. debug_buf_lvl - debug level at which buffering is turned off. tls_level - TLS security level (default: "tls_compat"). The recognized values for tls_level are: tls_secure: accept only TLS protocols recognized as "secure" tls_no_ssl: disable SSLv2 and SSLv3 support tls_compat: accept all SSL/TLS versions For more documentation see the docstring of the parent class IMAP4. """ def __init__(self, host=None, port=None, keyfile=None, certfile=None, ca_certs=None, cert_verify_cb=None, ssl_version="ssl23", debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None, tls_level=TLS_COMPAT): self.keyfile = keyfile self.certfile = certfile self.ca_certs = ca_certs self.cert_verify_cb = cert_verify_cb self.ssl_version = ssl_version self.tls_level = tls_level IMAP4.__init__(self, host, port, debug, debug_file, identifier, timeout, debug_buf_lvl) def open(self, host=None, port=None): """open(host=None, port=None) Setup secure connection to remote server on "host:port" (default: localhost:standard IMAP4 SSL port). This connection will be used by the routines: read, send, shutdown, socket, ssl.""" self.host = self._choose_nonull_or_dflt('', host) self.port = self._choose_nonull_or_dflt(IMAP4_SSL_PORT, port) self.sock = self.open_socket() self.ssl_wrap_socket() def read(self, size): """data = read(size) Read at most 'size' bytes from remote.""" if self.decompressor is None: return self.sock.read(size) if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: data = self.sock.read(READ_SIZE) return self.decompressor.decompress(data, size) def send(self, data): """send(data) Send 'data' to remote.""" if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: data = bytes(data, 'utf8') if hasattr(self.sock, "sendall"): self.sock.sendall(data) else: dlen = len(data) while dlen > 0: sent = self.sock.write(data) if sent == dlen: break # avoid copy data = data[sent:] dlen = dlen - sent def ssl(self): """ssl = ssl() Return ssl instance used to communicate with the IMAP4 server.""" return self.sock class IMAP4_stream(IMAP4): """IMAP4 client class over a stream Instantiate with: IMAP4_stream(command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None) command - string that can be passed to subprocess.Popen(); debug - debug level (default: 0 - no debug); debug_file - debug stream (default: sys.stderr); identifier - thread identifier prefix (default: host); timeout - timeout in seconds when expecting a command response. debug_buf_lvl - debug level at which buffering is turned off. For more documentation see the docstring of the parent class IMAP4. """ def __init__(self, command, debug=None, debug_file=None, identifier=None, timeout=None, debug_buf_lvl=None): self.command = command self.host = command self.port = None self.sock = None self.writefile, self.readfile = None, None self.read_fd = None IMAP4.__init__(self, None, None, debug, debug_file, identifier, timeout, debug_buf_lvl) def open(self, host=None, port=None): """open(host=None, port=None) Setup a stream connection via 'self.command'. This connection will be used by the routines: read, send, shutdown, socket.""" from subprocess import Popen, PIPE if __debug__: self._log(0, 'opening stream from command "%s"' % self.command) self._P = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True) self.writefile, self.readfile = self._P.stdin, self._P.stdout self.read_fd = self.readfile.fileno() def read(self, size): """Read 'size' bytes from remote.""" if self.decompressor is None: return os.read(self.read_fd, size) if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: data = os.read(self.read_fd, READ_SIZE) return self.decompressor.decompress(data, size) def send(self, data): """Send data to remote.""" if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) if bytes != str: data = bytes(data, 'utf8') self.writefile.write(data) self.writefile.flush() def shutdown(self): """Close I/O established in "open".""" self.readfile.close() self.writefile.close() class _Authenticator(object): """Private class to provide en/de-coding for base64 authentication conversation.""" def __init__(self, mechinst): self.mech = mechinst # Callable object to provide/process data def process(self, data, rqb): ret = self.mech(self.decode(data)) if ret is None: return '*' # Abort conversation return self.encode(ret) def encode(self, inp): # # Invoke binascii.b2a_base64 iteratively with # short even length buffers, strip the trailing # line feed from the result and append. "Even" # means a number that factors to both 6 and 8, # so when it gets to the end of the 8-bit input # there's no partial 6-bit output. # if bytes != str: oup = b'' else: oup = '' while inp: if len(inp) > 48: t = inp[:48] inp = inp[48:] else: t = inp inp = '' e = binascii.b2a_base64(t) if e: oup = oup + e[:-1] return oup def decode(self, inp): if not inp: return '' return binascii.a2b_base64(inp) class _IdleCont(object): """When process is called, server is in IDLE state and will send asynchronous changes.""" def __init__(self, parent, timeout): self.parent = parent self.timeout = parent._choose_nonull_or_dflt(IDLE_TIMEOUT, timeout) self.parent.idle_timeout = self.timeout + time.time() def process(self, data, rqb): self.parent.idle_lock.acquire() self.parent.idle_rqb = rqb self.parent.idle_timeout = self.timeout + time.time() self.parent.idle_lock.release() if __debug__: self.parent._log(2, 'server IDLE started, timeout in %.2f secs' % self.timeout) return None MonthNames = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] Mon2num = dict(list(zip((x for x in MonthNames[1:]), list(range(1, 13))))) InternalDate = re.compile(r'.*INTERNALDATE "' r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' r' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' r' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' r'"') def Internaldate2Time(resp): """time_tuple = Internaldate2Time(resp) Convert IMAP4 INTERNALDATE to UT.""" mo = InternalDate.match(resp) if not mo: return None mon = Mon2num[mo.group('mon')] zonen = mo.group('zonen') day = int(mo.group('day')) year = int(mo.group('year')) hour = int(mo.group('hour')) min = int(mo.group('min')) sec = int(mo.group('sec')) zoneh = int(mo.group('zoneh')) zonem = int(mo.group('zonem')) # INTERNALDATE timezone must be subtracted to get UT zone = (zoneh*60 + zonem)*60 if zonen == '-': zone = -zone tt = (year, mon, day, hour, min, sec, -1, -1, -1) utc = time.mktime(tt) # Following is necessary because the time module has no 'mkgmtime'. # 'mktime' assumes arg in local timezone, so adds timezone/altzone. lt = time.localtime(utc) if time.daylight and lt[-1]: zone = zone + time.altzone else: zone = zone + time.timezone return time.localtime(utc - zone) Internaldate2tuple = Internaldate2Time # (Backward compatible) def Time2Internaldate(date_time): """'"DD-Mmm-YYYY HH:MM:SS +HHMM"' = Time2Internaldate(date_time) Convert 'date_time' to IMAP4 INTERNALDATE representation.""" if isinstance(date_time, (int, float)): tt = time.localtime(date_time) elif isinstance(date_time, (tuple, time.struct_time)): tt = date_time elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): return date_time # Assume in correct format else: raise ValueError("date_time not of a known type") if time.daylight and tt[-1]: zone = -time.altzone else: zone = -time.timezone return ('"%2d-%s-%04d %02d:%02d:%02d %+03d%02d"' % ((tt[2], MonthNames[tt[1]], tt[0]) + tt[3:6] + divmod(zone//60, 60))) FLAGS_cre = re.compile(r'.*FLAGS \((?P[^\)]*)\)') def ParseFlags(resp): """('flag', ...) = ParseFlags(line) Convert IMAP4 flags response to python tuple.""" mo = FLAGS_cre.match(resp) if not mo: return () return tuple(mo.group('flags').split()) if __name__ == '__main__': # To test: invoke either as 'python imaplib2.py [IMAP4_server_hostname]', # or as 'python imaplib2.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' # or as 'python imaplib2.py -l keyfile[:certfile]|: [IMAP4_SSL_server_hostname]' # # Option "-d " turns on debugging (use "-d 5" for everything) # Option "-i" tests that IDLE is interruptible # Option "-p " allows alternate ports if not __debug__: raise ValueError('Please run without -O') import getopt, getpass try: optlist, args = getopt.getopt(sys.argv[1:], 'd:il:s:p:') except getopt.error as val: optlist, args = (), () debug, debug_buf_lvl, port, stream_command, keyfile, certfile, idle_intr = (None,)*7 for opt,val in optlist: if opt == '-d': debug = int(val) debug_buf_lvl = debug - 1 elif opt == '-i': idle_intr = 1 elif opt == '-l': try: keyfile,certfile = val.split(':') except ValueError: keyfile,certfile = val,val elif opt == '-p': port = int(val) elif opt == '-s': stream_command = val if not args: args = (stream_command,) if not args: args = ('',) if not port: port = (keyfile is not None) and IMAP4_SSL_PORT or IMAP4_PORT host = args[0] USER = getpass.getuser() data = open(os.path.exists("test.data") and "test.data" or __file__).read(1000) test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)s%(data)s' \ % {'user':USER, 'lf':'\n', 'data':data} test_seq1 = [ ('list', ('""', '""')), ('list', ('""', '%')), ('create', ('imaplib2_test0',)), ('rename', ('imaplib2_test0', 'imaplib2_test1')), ('CREATE', ('imaplib2_test2',)), ('append', ('imaplib2_test2', None, None, test_mesg)), ('list', ('', 'imaplib2_test%')), ('select', ('imaplib2_test2',)), ('search', (None, 'SUBJECT', 'IMAP4 test')), ('fetch', ("'1:*'", '(FLAGS INTERNALDATE RFC822)')), ('store', ('1', 'FLAGS', '(\Deleted)')), ('namespace', ()), ('expunge', ()), ('recent', ()), ('close', ()), ] test_seq2 = ( ('select', ()), ('response', ('UIDVALIDITY',)), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), ('examine', ()), ('select', ()), ('fetch', ("'1:*'", '(FLAGS UID)')), ('examine', ()), ('select', ()), ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')), ('uid', ('SEARCH', 'ALL')), ('uid', ('THREAD', 'references', 'UTF-8', '(SEEN)')), ('recent', ()), ) AsyncError, M = None, None def responder(cb_arg_list): response, cb_arg, error = cb_arg_list global AsyncError cmd, args = cb_arg if error is not None: AsyncError = error M._log(0, '[cb] ERROR %s %.100s => %s' % (cmd, args, error)) return typ, dat = response M._log(0, '[cb] %s %.100s => %s %.100s' % (cmd, args, typ, dat)) if typ == 'NO': AsyncError = (Exception, dat[0]) def run(cmd, args, cb=True): if AsyncError: M._log(1, 'AsyncError %s' % repr(AsyncError)) M.logout() typ, val = AsyncError raise typ(val) if not M.debug: M._log(0, '%s %.100s' % (cmd, args)) try: if cb: typ, dat = getattr(M, cmd)(callback=responder, cb_arg=(cmd, args), *args) M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) else: typ, dat = getattr(M, cmd)(*args) M._log(1, '%s %.100s => %s %.100s' % (cmd, args, typ, dat)) except: M._log(1, '%s - %s' % sys.exc_info()[:2]) M.logout() raise if typ == 'NO': M._log(1, 'NO') M.logout() raise Exception(dat[0]) return dat try: threading.currentThread().setName('main') if keyfile is not None: if not keyfile: keyfile = None if not certfile: certfile = None M = IMAP4_SSL(host=host, port=port, keyfile=keyfile, certfile=certfile, ssl_version="tls1", debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl, tls_level="tls_no_ssl") elif stream_command: M = IMAP4_stream(stream_command, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) else: M = IMAP4(host=host, port=port, debug=debug, identifier='', timeout=10, debug_buf_lvl=debug_buf_lvl) if M.state != 'AUTH': # Login needed PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) test_seq1.insert(0, ('login', (USER, PASSWD))) M._log(0, 'PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) if 'COMPRESS=DEFLATE' in M.capabilities: M.enable_compression() for cmd,args in test_seq1: run(cmd, args) for ml in run('list', ('', 'imaplib2_test%'), cb=False): mo = re.match(r'.*"([^"]+)"$', ml) if mo: path = mo.group(1) else: path = ml.split()[-1] run('delete', (path,)) if 'ID' in M.capabilities: run('id', ()) run('id', ("(name imaplib2)",)) run('id', ("version", __version__, "os", os.uname()[0])) for cmd,args in test_seq2: if (cmd,args) != ('uid', ('SEARCH', 'SUBJECT', 'IMAP4 test')): run(cmd, args) continue dat = run(cmd, args, cb=False) uid = dat[-1].split() if not uid: continue run('uid', ('FETCH', uid[-1], '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) run('uid', ('STORE', uid[-1], 'FLAGS', '(\Deleted)')) run('expunge', ()) if 'IDLE' in M.capabilities: run('idle', (2,), cb=False) run('idle', (99,)) # Asynchronous, to test interruption of 'idle' by 'noop' time.sleep(1) run('noop', (), cb=False) run('append', (None, None, None, test_mesg), cb=False) num = run('search', (None, 'ALL'), cb=False)[0].split()[0] dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) M._mesg('fetch %s => %s' % (num, repr(dat))) run('idle', (2,)) run('store', (num, '-FLAGS', '(\Seen)'), cb=False), dat = run('fetch', (num, '(FLAGS INTERNALDATE RFC822)'), cb=False) M._mesg('fetch %s => %s' % (num, repr(dat))) run('uid', ('STORE', num, 'FLAGS', '(\Deleted)')) run('expunge', ()) if idle_intr: M._mesg('HIT CTRL-C to interrupt IDLE') try: run('idle', (99,), cb=False) # Synchronous, to test interruption of 'idle' by INTR except KeyboardInterrupt: M._mesg('Thanks!') M._mesg('') raise elif idle_intr: M._mesg('chosen server does not report IDLE capability') run('logout', (), cb=False) if debug: M._mesg('') M._print_log() M._mesg('') M._mesg('unused untagged responses in order, most recent last:') for typ,dat in M.pop_untagged_responses(): M._mesg('\t%s %s' % (typ, dat)) print('All tests OK.') except: if not idle_intr or M is None or not 'IDLE' in M.capabilities: print('Tests failed.') if not debug: print(''' If you would like to see debugging output, try: %s -d5 ''' % sys.argv[0]) raise ================================================ FILE: offlineimap/emailutil.py ================================================ # Some useful functions to extract data out of emails # Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import email from email.parser import Parser as MailParser def get_message_date(content, header='Date'): """Parses mail and returns resulting timestamp. :param header: the header to extract date from; :returns: timestamp or `None` in the case of failure. """ message = MailParser().parsestr(content, True) dateheader = message.get(header) # parsedate_tz returns a 10-tuple that can be passed to mktime_tz # Will be None if missing or not in a valid format. Note that # indexes 6, 7, and 8 of the result tuple are not usable. datetuple = email.utils.parsedate_tz(dateheader) if datetuple is None: return None return email.utils.mktime_tz(datetuple) ================================================ FILE: offlineimap/error.py ================================================ class OfflineImapError(Exception): """An Error during offlineimap synchronization""" class ERROR: """Severity level of an Exception * **MESSAGE**: Abort the current message, but continue with folder * **FOLDER_RETRY**: Error syncing folder, but do retry * **FOLDER**: Abort folder sync, but continue with next folder * **REPO**: Abort repository sync, continue with next account * **CRITICAL**: Immediately exit offlineimap """ MESSAGE, FOLDER_RETRY, FOLDER, REPO, CRITICAL = 0, 10, 15, 20, 30 def __init__(self, reason, severity, errcode=None): """ :param reason: Human readable string suitable for logging :param severity: denoting which operations should be aborted. E.g. a ERROR.MESSAGE can occur on a faulty message, but a ERROR.REPO occurs when the server is offline. :param errcode: optional number denoting a predefined error situation (which let's us exit with a predefined exit value). So far, no errcodes have been defined yet. :type severity: OfflineImapError.ERROR value""" self.errcode = errcode self.severity = severity # 'reason' is stored in the Exception().args tuple. super(OfflineImapError, self).__init__(reason) @property def reason(self): return self.args[0] ================================================ FILE: offlineimap/folder/Base.py ================================================ # Base folder support # Copyright (C) 2002-2016 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os.path import re import time from sys import exc_info from offlineimap import threadutil from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError import offlineimap.accounts class BaseFolder(object): __hash__ = None def __init__(self, name, repository): """ :param name: Path & name of folder minus root or reference :param repository: Repository() in which the folder is. """ self.ui = getglobalui() self.messagelist = {} # Save original name for folderfilter operations. self.ffilter_name = name # Top level dir name is always ''. self.root = None self.name = name if not name == self.getsep() else '' self.newmail_hook = None # Only set the newmail_hook if the IMAP folder is named 'INBOX'. if self.name == 'INBOX': self.newmail_hook = repository.newmail_hook self.have_newmail = False self.copy_ignoreUIDs = None # List of UIDs to ignore. self.repository = repository self.visiblename = repository.nametrans(name) # In case the visiblename becomes '.' or '/' (top-level) we use # '' as that is the name that e.g. the Maildir scanning will # return for the top-level dir. if self.visiblename == self.getsep(): self.visiblename = '' self.repoconfname = "Repository " + repository.name self.config = repository.getconfig() # Do we need to use mail timestamp for filename prefix? filename_use_mail_timestamp_global = self.config.getdefaultboolean( "general", "filename_use_mail_timestamp", False) self._filename_use_mail_timestamp = self.config.getdefaultboolean( self.repoconfname, "filename_use_mail_timestamp", filename_use_mail_timestamp_global) self._sync_deletes = self.config.getdefaultboolean( self.repoconfname, "sync_deletes", True) self._dofsync = self.config.getdefaultboolean("general", "fsync", True) # Determine if we're running static or dynamic folder filtering # and check filtering status. self._dynamic_folderfilter = self.config.getdefaultboolean( self.repoconfname, "dynamic_folderfilter", False) self._sync_this = repository.should_sync_folder(self.ffilter_name) if self._dynamic_folderfilter: self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]"% (self.ffilter_name, repository)) elif not self._sync_this: self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter"% (self.ffilter_name, repository)) # Passes for syncmessagesto. self.syncmessagesto_passes = [ self.__syncmessagesto_copy, self.__syncmessagesto_delete, self.__syncmessagesto_flags, ] def getname(self): """Returns name""" return self.name def __str__(self): # FIMXE: remove calls of this. We have getname(). return self.name def __unicode__(self): # NOTE(sheeprine): Implicit call to this by UIBase deletingflags() which # fails if the str is utf-8 return self.name.decode('utf-8') def __enter__(self): """Starts a transaction. This will postpone (guaranteed) saving to disk of all messages saved inside this transaction until its committed.""" pass def __exit__(self, exc_type, exc_val, exc_tb): """Commits a transaction, all messages saved inside this transaction will only now be persisted to disk.""" pass @property def accountname(self): """Account name as string""" return self.repository.accountname @property def sync_this(self): """Should this folder be synced or is it e.g. filtered out?""" if not self._dynamic_folderfilter: return self._sync_this else: return self.repository.should_sync_folder(self.ffilter_name) def dofsync(self): return self._dofsync def suggeststhreads(self): """Returns True if this folder suggests using threads for actions. Only IMAP returns True. This method must honor any CLI or configuration option.""" return False def waitforthread(self): """Implements method that waits for thread to be usable. Should be implemented only for folders that suggest threads.""" raise NotImplementedError def quickchanged(self, statusfolder): """ Runs quick check for folder changes and returns changed status: True -- changed, False -- not changed. :param statusfolder: keeps track of the last known folder state. """ return True def getinstancelimitnamespace(self): """For threading folders, returns the instancelimitname for InstanceLimitedThreads.""" raise NotImplementedError def storesmessages(self): """Should be true for any backend that actually saves message bodies. (Almost all of them). False for the LocalStatus backend. Saves us from having to slurp up messages just for localstatus purposes.""" return 1 def getvisiblename(self): """The nametrans-transposed name of the folder's name.""" return self.visiblename def getexplainedname(self): """Name that shows both real and nametrans-mangled values.""" if self.name == self.visiblename: return self.name else: return "%s [remote name %s]"% (self.visiblename, self.name) def getrepository(self): """Returns the repository object that this folder is within.""" return self.repository def getroot(self): """Returns the root of the folder, in a folder-specific fashion.""" return self.root def getsep(self): """Returns the separator for this folder type.""" return self.sep def getfullname(self): if self.getroot(): return self.getroot() + self.getsep() + self.getname() else: return self.getname() def getfolderbasename(self): """Return base file name of file to store Status/UID info in.""" if not self.name: basename = '.' else: # Avoid directory hierarchies and file names such as '/'. basename = self.name.replace('/', '.') # Replace with literal 'dot' if final path name is '.' as '.' is # an invalid file name. basename = re.sub('(^|\/)\.$', '\\1dot', basename) return basename def check_uidvalidity(self): """Tests if the cached UIDVALIDITY match the real current one If required it saves the UIDVALIDITY value. In this case the function is not threadsafe. So don't attempt to call it from concurrent threads. :returns: Boolean indicating the match. Returns True in case it implicitely saved the UIDVALIDITY.""" if self.get_saveduidvalidity() != None: return self.get_saveduidvalidity() == self.get_uidvalidity() else: self.save_uidvalidity() return True def _getuidfilename(self): """provides UIDVALIDITY cache filename for class internal purposes.""" return os.path.join(self.repository.getuiddir(), self.getfolderbasename()) def get_saveduidvalidity(self): """Return the previously cached UIDVALIDITY value :returns: UIDVALIDITY as (long) number or None, if None had been saved yet.""" if hasattr(self, '_base_saved_uidvalidity'): return self._base_saved_uidvalidity uidfilename = self._getuidfilename() if not os.path.exists(uidfilename): self._base_saved_uidvalidity = None else: file = open(uidfilename, "rt") self._base_saved_uidvalidity = int(file.readline().strip()) file.close() return self._base_saved_uidvalidity def save_uidvalidity(self): """Save the UIDVALIDITY value of the folder to the cache This function is not threadsafe, so don't attempt to call it from concurrent threads.""" newval = self.get_uidvalidity() uidfilename = self._getuidfilename() with open(uidfilename + ".tmp", "wt") as uidfile: uidfile.write("%d\n"% newval) os.rename(uidfilename + ".tmp", uidfilename) self._base_saved_uidvalidity = newval def get_uidvalidity(self): """Retrieve the current connections UIDVALIDITY value This function needs to be implemented by each Backend :returns: UIDVALIDITY as a (long) number.""" raise NotImplementedError def cachemessagelist(self): """Cache the list of messages. Reads the message list from disk or network and stores it in memory for later use. This list will not be re-read from disk or memory unless this function is called again.""" raise NotImplementedError def ismessagelistempty(self): """Is the list of messages empty.""" if len(self.messagelist.keys()) < 1: return True return False def dropmessagelistcache(self): """Empty everythings we know about messages.""" self.messagelist = {} def getmessagelist(self): """Gets the current message list. You must call cachemessagelist() before calling this function!""" return self.messagelist def msglist_item_initializer(self, uid): """Returns value for empty messagelist element with given UID. This function must initialize all fields of messagelist item and must be called every time when one creates new messagelist entry to ensure that all fields that must be present are present.""" raise NotImplementedError def uidexists(self, uid): """Returns True if uid exists.""" return uid in self.getmessagelist() def getmessageuidlist(self): """Gets a list of UIDs. You may have to call cachemessagelist() before calling this function!""" return sorted(self.getmessagelist().keys()) def getmessagecount(self): """Gets the number of messages.""" return len(self.getmessagelist()) def getmessage(self, uid): """Returns the content of the specified message.""" raise NotImplementedError def getmaxage(self): """Return maxage. maxage is allowed to be either an integer or a date of the form YYYY-mm-dd. This returns a time_struct.""" maxagestr = self.config.getdefault("Account %s"% self.accountname, "maxage", None) if maxagestr is None: return None # Is it a number? try: maxage = int(maxagestr) if maxage < 1: raise OfflineImapError("invalid maxage value %d"% maxage, OfflineImapError.ERROR.MESSAGE) return time.gmtime(time.time() - 60*60*24*maxage) except ValueError: pass # Maybe it was a date. # Is it a date string? try: date = time.strptime(maxagestr, "%Y-%m-%d") if date[0] < 1900: raise OfflineImapError("maxage led to year %d. " "Abort syncing."% date[0], OfflineImapError.ERROR.MESSAGE) if (time.mktime(date) - time.mktime(time.localtime())) > 0: raise OfflineImapError("maxage led to future date %s. " "Abort syncing."% maxagestr, OfflineImapError.ERROR.MESSAGE) return date except ValueError: raise OfflineImapError("invalid maxage value %s"% maxagestr, OfflineImapError.ERROR.MESSAGE) def getmaxsize(self): return self.config.getdefaultint("Account %s"% self.accountname, "maxsize", None) def getstartdate(self): """ Retrieve the value of the configuration option startdate """ datestr = self.config.getdefault("Repository " + self.repository.name, 'startdate', None) try: if not datestr: return None date = time.strptime(datestr, "%Y-%m-%d") if date[0] < 1900: raise OfflineImapError("startdate led to year %d. " "Abort syncing."% date[0], OfflineImapError.ERROR.MESSAGE) if (time.mktime(date) - time.mktime(time.localtime())) > 0: raise OfflineImapError("startdate led to future date %s. " "Abort syncing."% datestr, OfflineImapError.ERROR.MESSAGE) return date except ValueError: raise OfflineImapError("invalid startdate value %s", OfflineImapError.ERROR.MESSAGE) def get_min_uid_file(self): startuiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.repository.name, 'StartUID') if not os.path.exists(startuiddir): os.mkdir(startuiddir, 0o700) return os.path.join(startuiddir, self.getfolderbasename()) def save_min_uid(self, min_uid): uidfile = self.get_min_uid_file() fd = open(uidfile, 'wt') fd.write(str(min_uid) + "\n") fd.close() def retrieve_min_uid(self): uidfile = self.get_min_uid_file() if not os.path.exists(uidfile): return None try: fd = open(uidfile, 'rt') min_uid = int(fd.readline().strip()) fd.close() return min_uid except: raise IOError("Can't read %s"% uidfile) def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. If the uid is < 0: The backend should assign a new uid and return it. In case it cannot assign a new uid, it returns the negative uid passed in WITHOUT saving the message. If the backend CAN assign a new uid, but cannot find out what this UID is (as is the case with some IMAP servers), it returns 0 but DOES save the message. IMAP backend should be the only one that can assign a new uid. If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" raise NotImplementedError def getmessagetime(self, uid): """Return the received time for the specified message.""" raise NotImplementedError def getmessagemtime(self, uid): """Returns the message modification time of the specified message.""" raise NotImplementedError def getmessageflags(self, uid): """Returns the flags for the specified message.""" raise NotImplementedError def getmessagekeywords(self, uid): """Returns the keywords for the specified message.""" raise NotImplementedError def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" raise NotImplementedError def addmessageflags(self, uid, flags): """Adds the specified flags to the message's flag set. If a given flag is already present, it will not be duplicated. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. :param flags: A set() of flags""" newflags = self.getmessageflags(uid) | flags self.savemessageflags(uid, newflags) def addmessagesflags(self, uidlist, flags): """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" for uid in uidlist: if self.uidexists(uid): self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. If a given flag is already removed, no action will be taken for that flag.""" newflags = self.getmessageflags(uid) - flags self.savemessageflags(uid, newflags) def deletemessagesflags(self, uidlist, flags): """ Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" for uid in uidlist: self.deletemessageflags(uid, flags) def getmessagelabels(self, uid): """Returns the labels for the specified message.""" raise NotImplementedError def savemessagelabels(self, uid, labels, ignorelabels=set(), mtime=0): """Sets the specified message's labels to the given set. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" raise NotImplementedError def addmessagelabels(self, uid, labels): """Adds the specified labels to the message's labels set. If a given label is already present, it will not be duplicated. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. :param labels: A set() of labels""" newlabels = self.getmessagelabels(uid) | labels self.savemessagelabels(uid, newlabels) def addmessageslabels(self, uidlist, labels): """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" for uid in uidlist: self.addmessagelabels(uid, labels) def deletemessagelabels(self, uid, labels): """Removes each label given from the message's label set. If a given label is already removed, no action will be taken for that label. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" newlabels = self.getmessagelabels(uid) - labels self.savemessagelabels(uid, newlabels) def deletemessageslabels(self, uidlist, labels): """ Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" for uid in uidlist: self.deletemessagelabels(uid, labels) def addmessageheader(self, content, linebreak, headername, headervalue): """Adds new header to the provided message. WARNING: This function is a bit tricky, and modifying it in the wrong way, may easily lead to data-loss. Arguments: - content: message content, headers and body as a single string - linebreak: string that carries line ending - headername: name of the header to add - headervalue: value of the header to add .. note:: The following documentation will not get displayed correctly after being processed by Sphinx. View the source of this method to read it. This has to deal with strange corner cases where the header is missing or empty. Here are illustrations for all the cases, showing where the header gets inserted and what the end result is. In each illustration, '+' means the added contents. Note that these examples assume LF for linebreak, not CRLF, so '\n' denotes a linebreak and '\n\n' corresponds to the transition between header and body. However if the linebreak parameter is set to '\r\n' then you would have to substitute '\r\n' for '\n' in the below examples. * Case 1: No '\n\n', leading '\n' +X-Flying-Pig-Header: i am here\n \n This is the body\n next line\n * Case 2: '\n\n' at position 0 +X-Flying-Pig-Header: i am here \n \n This is the body\n next line\n * Case 3: No '\n\n', no leading '\n' +X-Flying-Pig-Header: i am here\n +\n This is the body\n next line\n * Case 4: '\n\n' at non-zero position Subject: Something wrong with OI\n From: some@person.at +\nX-Flying-Pig-Header: i am here \n \n This is the body\n next line\n """ self.ui.debug('', 'addmessageheader: called to add %s: %s'% (headername, headervalue)) insertionpoint = content.find(linebreak * 2) if insertionpoint == -1: self.ui.debug('', 'addmessageheader: headers were missing') else: self.ui.debug('', 'addmessageheader: headers end at position %d'% insertionpoint) mark = '==>EOH<==' contextstart = max(0, insertionpoint - 100) contextend = min(len(content), insertionpoint + 100) self.ui.debug('', 'addmessageheader: header/body transition " \ "context (marked by %s): %s%s%s'% ( mark, repr(content[contextstart:insertionpoint]), mark, repr(content[insertionpoint:contextend]) ) ) # Hoping for case #4. prefix = linebreak suffix = '' # Case #2. if insertionpoint == 0: prefix = '' suffix = '' # Either case #1 or #3. elif insertionpoint == -1: prefix = '' suffix = linebreak insertionpoint = 0 # Case #3: when body starts immediately, without preceding '\n' # (this shouldn't happen with proper mail messages, but # we seen many broken ones), we should add '\n' to make # new (and the only header, in this case) to be properly # separated from the message body. if content[0:len(linebreak)] != linebreak: suffix = suffix + linebreak self.ui.debug('', 'addmessageheader: insertionpoint = %d'% insertionpoint) headers = content[0:insertionpoint] self.ui.debug('', 'addmessageheader: headers = %s'% repr(headers)) new_header = prefix + ("%s: %s"% (headername, headervalue)) + suffix self.ui.debug('', 'addmessageheader: new_header = %s'% repr(new_header)) return headers + new_header + content[insertionpoint:] def __find_eoh(self, content): """Searches for the point where mail headers end. Either double '\n', or end of string. Arguments: - content: contents of the message to search in Returns: position of the first non-header byte. """ eoh_cr = content.find('\n\n') if eoh_cr == -1: eoh_cr = len(content) return eoh_cr def getmessageheader(self, content, name): """Return the value of the first occurence of the given header. Header name is case-insensitive. Arguments: - contents: message itself - name: name of the header to be searched Returns: header value or None if no such header was found. """ self.ui.debug('', 'getmessageheader: called to get %s'% name) eoh = self.__find_eoh(content) self.ui.debug('', 'getmessageheader: eoh = %d'% eoh) headers = content[0:eoh] self.ui.debug('', 'getmessageheader: headers = %s'% repr(headers)) m = re.search('^%s:(.*)$'% name, headers, flags= re.MULTILINE | re.IGNORECASE) if m: return m.group(1).strip() else: return None def getmessageheaderlist(self, content, name): """Return a list of values for the given header. Arguments: - contents: message itself - name: name of the header to be searched Returns: list of header values or empty list if no such header was found. """ self.ui.debug('', 'getmessageheaderlist: called to get %s'% name) eoh = self.__find_eoh(content) self.ui.debug('', 'getmessageheaderlist: eoh = %d'% eoh) headers = content[0:eoh] self.ui.debug('', 'getmessageheaderlist: headers = %s'% repr(headers)) return re.findall('^%s:(.*)$'% name, headers, flags= re.MULTILINE | re.IGNORECASE) def deletemessageheaders(self, content, header_list): """Deletes headers in the given list from the message content. Arguments: - content: message itself - header_list: list of headers to be deleted or just the header name We expect our message to have '\n' as line endings.""" if type(header_list) != type([]): header_list = [header_list] self.ui.debug('', 'deletemessageheaders: called to delete %s'% (header_list)) if not len(header_list): return content eoh = self.__find_eoh(content) self.ui.debug('', 'deletemessageheaders: end of headers = %d'% eoh) headers = content[0:eoh] rest = content[eoh:] self.ui.debug('', 'deletemessageheaders: headers = %s'% repr(headers)) new_headers = [] for h in headers.split('\n'): keep_it = True for trim_h in header_list: if len(h) > len(trim_h) and h[0:len(trim_h)+1] == (trim_h + ":"): keep_it = False break if keep_it: new_headers.append(h) return '\n'.join(new_headers) + rest def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid. If the backend supports it (IMAP does not). :param new_uid: (optional) If given, the old UID will be changed to a new UID. This allows backends efficient renaming of messages if the UID has changed.""" raise NotImplementedError def deletemessage(self, uid): """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" raise NotImplementedError def deletemessages(self, uidlist): """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" for uid in uidlist: self.deletemessage(uid) def copymessageto(self, uid, dstfolder, statusfolder, register=1): """Copies a message from self to dst if needed, updating the status Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. :param uid: uid of the message to be copied. :param dstfolder: A BaseFolder-derived instance :param statusfolder: A LocalStatusFolder instance :param register: whether we should register a new thread." :returns: Nothing on success, or raises an Exception.""" # Sometimes, it could be the case that if a sync takes awhile, # a message might be deleted from the maildir before it can be # synced to the status cache. This is only a problem with # self.getmessage(). So, don't call self.getmessage unless # really needed. if register: # Output that we start a new thread. self.ui.registerthread(self.repository.account) try: message = None flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) # If any of the destinations actually stores the message body, # load it up. if dstfolder.storesmessages(): message = self.getmessage(uid) # Succeeded? -> IMAP actually assigned a UID. If newid # remained negative, no server was willing to assign us an # UID. If newid is 0, saving succeeded, but we could not # retrieve the new UID. Ignore message in this case. new_uid = dstfolder.savemessage(uid, message, flags, rtime) if new_uid > 0: if new_uid != uid: # Got new UID, change the local uid to match the new one. self.change_message_uid(uid, new_uid) statusfolder.deletemessage(uid) # Got new UID, change the local uid. # Save uploaded status in the statusfolder. statusfolder.savemessage(new_uid, message, flags, rtime) # Check whether the mail has been seen. if 'S' not in flags: self.have_newmail = True elif new_uid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created # in IMAP. So we just delete local message and on next run # we'll sync it back # XXX This could cause infinite loop on syncing between two # IMAP servers ... self.deletemessage(uid) else: raise OfflineImapError("Trying to save msg (uid %d) on folder " "%s returned invalid uid %d"% (uid, dstfolder.getvisiblename(), new_uid), OfflineImapError.ERROR.MESSAGE) except (KeyboardInterrupt): # Bubble up CTRL-C. raise except OfflineImapError as e: if e.severity > OfflineImapError.ERROR.MESSAGE: raise # Bubble severe errors up. self.ui.error(e, exc_info()[2]) except Exception as e: self.ui.error(e, exc_info()[2], msg = "Copying message %s [acc: %s]"% (uid, self.accountname)) raise # Raise on unknown errors, so we can fix those. def __syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side. This will copy messages to dstfolder that exist locally but are not in the statusfolder yet. The strategy is: 1) Look for messages present in self but not in statusfolder. 2) invoke copymessageto() on those which: - If dstfolder doesn't have it yet, add them to dstfolder. - Update statusfolder. This function checks and protects us from action in dryrun mode.""" # We have no new mail yet. self.have_newmail = False threads = [] copylist = [uid for uid in self.getmessageuidlist() if not statusfolder.uidexists(uid)] num_to_copy = len(copylist) # Honor 'copy_ignore_eval' configuration option. if self.copy_ignoreUIDs is not None: for uid in self.copy_ignoreUIDs: if uid in copylist: copylist.remove(uid) self.ui.ignorecopyingmessage(uid, self, dstfolder) if num_to_copy > 0 and self.repository.account.dryrun: self.ui.info("[DRYRUN] Copy {} messages from {}[{}] to {}".format( num_to_copy, self, self.repository, dstfolder.repository) ) return with self: for num, uid in enumerate(copylist): # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break if uid == 0: self.ui.warn("Assertion that UID != 0 failed; ignoring message.") continue if uid > 0 and dstfolder.uidexists(uid): # dstfolder has message with that UID already, only update status. flags = self.getmessageflags(uid) rtime = self.getmessagetime(uid) statusfolder.savemessage(uid, None, flags, rtime) continue self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) # Exceptions are caught in copymessageto(). if self.suggeststhreads(): self.waitforthread() thread = threadutil.InstanceLimitedThread( self.getinstancelimitnamespace(), target=self.copymessageto, name="Copy message from %s:%s"% (self.repository, self), args=(uid, dstfolder, statusfolder) ) thread.start() threads.append(thread) else: self.copymessageto(uid, dstfolder, statusfolder, register=0) for thread in threads: thread.join() # Block until all "copy" threads are done. # Execute new mail hook if we have new mail. if self.have_newmail: if self.newmail_hook != None: self.newmail_hook() def __syncmessagesto_delete(self, dstfolder, statusfolder): """Pass 2: Remove locally deleted messages on dst. Get all UIDs in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and statusfolder. This function checks and protects us from action in dryrun mode. """ # The list of messages to delete. If sync of deletions is disabled we # still remove stale entries from statusfolder (neither in local nor # remote). deletelist = [uid for uid in statusfolder.getmessageuidlist() if uid >= 0 and not self.uidexists(uid) and (self._sync_deletes or not dstfolder.uidexists(uid))] if len(deletelist): # Delete in statusfolder first to play safe. In case of abort, we # won't lose message, we will just unneccessarily retransmit some. # Delete messages from statusfolder that were either deleted by the # user, or not being tracked (e.g. because of maxage). if not self.repository.account.dryrun: statusfolder.deletemessages(deletelist) # Filter out untracked messages. deletelist = [uid for uid in deletelist if dstfolder.uidexists(uid)] if len(deletelist): self.ui.deletingmessages(deletelist, [dstfolder]) if not self.repository.account.dryrun: dstfolder.deletemessages(deletelist) def combine_flags_and_keywords(self, uid, dstfolder): """Combine the message's flags and keywords using the mapping for the destination folder.""" # Take a copy of the message flag set, otherwise # __syncmessagesto_flags() will fail because statusflags is actually a # reference to selfflags (which it should not, but I don't have time to # debug THAT). selfflags = set(self.getmessageflags(uid)) try: keywordmap = dstfolder.getrepository().getkeywordmap() if keywordmap is None: return selfflags knownkeywords = set(keywordmap.keys()) selfkeywords = self.getmessagekeywords(uid) if not knownkeywords >= selfkeywords: # Some of the message's keywords are not in the mapping, so # skip them. skipped_keywords = list(selfkeywords - knownkeywords) selfkeywords &= knownkeywords self.ui.warn("Unknown keywords skipped: %s\n" "You may want to change your configuration to include " "those\n" % (skipped_keywords)) keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) # Add the mapped keywords to the list of message flags. selfflags |= keywordletterset except NotImplementedError: pass return selfflags def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization. Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to both dstfolder and statusfolder. This function checks and protects us from action in ryrun mode. """ # For each flag, we store a list of uids to which it should be # added. Then, we can call addmessagesflags() to apply them in # bulk, rather than one call per message. addflaglist = {} delflaglist = {} for uid in self.getmessageuidlist(): # Ignore messages with negative UIDs missed by pass 1 and # don't do anything if the message has been deleted remotely if uid < 0 or not dstfolder.uidexists(uid): continue if statusfolder.uidexists(uid): statusflags = statusfolder.getmessageflags(uid) else: statusflags = set() selfflags = self.combine_flags_and_keywords(uid, dstfolder) addflags = selfflags - statusflags delflags = statusflags - selfflags for flag in addflags: if not flag in addflaglist: addflaglist[flag] = [] addflaglist[flag].append(uid) for flag in delflags: if not flag in delflaglist: delflaglist[flag] = [] delflaglist[flag].append(uid) for flag, uids in addflaglist.items(): self.ui.addingflags(uids, flag, dstfolder) if self.repository.account.dryrun: continue # Don't actually add in a dryrun. dstfolder.addmessagesflags(uids, set(flag)) statusfolder.addmessagesflags(uids, set(flag)) for flag, uids in delflaglist.items(): self.ui.deletingflags(uids, flag, dstfolder) if self.repository.account.dryrun: continue # Don't actually remove in a dryrun. dstfolder.deletemessagesflags(uids, set(flag)) statusfolder.deletemessagesflags(uids, set(flag)) def syncmessagesto(self, dstfolder, statusfolder): """Syncs messages in this folder to the destination dstfolder. This is the high level entry for syncing messages in one direction. Syncsteps are: Pass1: Copy locally existing messages Copy messages in self, but not statusfolder to dstfolder if not already in dstfolder. dstfolder might assign a new UID (e.g. if uploading to IMAP). Update statusfolder. Pass2: Remove locally deleted messages Get all UIDS in statusfolder but not self. These are messages that were deleted in 'self'. Delete those from dstfolder and statusfolder. After this pass, the message lists should be identical wrt the uids present (except for potential negative uids that couldn't be placed anywhere). Pass3: Synchronize flag changes Compare flag mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the flag change to both dstfolder and statusfolder. Pass4: Synchronize label changes (Gmail only) Compares label mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder, syncs the labels to both dstfolder and statusfolder. :param dstfolder: Folderinstance to sync the msgs to. :param statusfolder: LocalStatus instance to sync against. """ for action in self.syncmessagesto_passes: # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break try: action(dstfolder, statusfolder) except (KeyboardInterrupt): raise except OfflineImapError as e: if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], "while syncing %s [account %s]"% (self, self.accountname)) except Exception as e: self.ui.error(e, exc_info()[2], "while syncing %s [account %s]"% (self, self.accountname)) raise # Raise unknown Exceptions so we can fix them. def __eq__(self, other): """Comparisons work either on string comparing folder names or on the same instance. MailDirFolder('foo') == 'foo' --> True a = MailDirFolder('foo'); a == b --> True MailDirFolder('foo') == 'moo' --> False MailDirFolder('foo') == IMAPFolder('foo') --> False MailDirFolder('foo') == MaildirFolder('foo') --> False """ if isinstance(other, str): return other == self.name return id(self) == id(other) def __ne__(self, other): return not self.__eq__(other) ================================================ FILE: offlineimap/folder/Gmail.py ================================================ # Gmail IMAP folder support # Copyright (C) 2002-2017 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """Folder implementation to support features of the Gmail IMAP server.""" import re import six from sys import exc_info from offlineimap import imaputil, imaplibutil, OfflineImapError import offlineimap.accounts from .IMAP import IMAPFolder class GmailFolder(IMAPFolder): """Folder implementation to support features of the Gmail IMAP server. Removing a message from a folder will only remove the "label" from the message and keep it in the "All mails" folder. To really delete a message it needs to be copied to the Trash folder. However, this is dangerous as our folder moves are implemented as a 1) delete in one folder and 2) append to the other. If 2 comes before 1, this will effectively delete the message from all folders. So we cannot do that until we have a smarter folder move mechanism. For more information on the Gmail IMAP server: http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 https://developers.google.com/google-apps/gmail/imap_extensions """ def __init__(self, imapserver, name, repository, decode=True): super(GmailFolder, self).__init__(imapserver, name, repository, decode) # The header under which labels are stored self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') # enables / disables label sync self.synclabels = self.repository.account.getconfboolean('synclabels', False) # if synclabels is enabled, add a 4th pass to sync labels if self.synclabels: self.imap_query.insert(0, 'X-GM-LABELS') self.syncmessagesto_passes.append(self.syncmessagesto_labels) # Labels to be left alone ignorelabels = self.repository.account.getconf('ignorelabels', '') self.ignorelabels = set([l for l in re.split(r'\s*,\s*', ignorelabels) if len(l)]) def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). Also gets Gmail labels and embeds them into the message. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ data = self._fetch_from_imap(str(uid), self.retrycount) # data looks now e.g. #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] # we only asked for one message, and that msg is in data[0]. # msbody is in [0][1]. body = data[0][1].replace("\r\n", "\n") # Embed the labels into the message headers if self.synclabels: m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0][0]) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) # First remove old label headers that may be in the message content retrieved # from gmail Then add a labels header with current gmail labels. body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: dbg_output = "%s...%s"% (str(body)[:150], str(body)[-50:]) else: dbg_output = body self.ui.debug('imap', "Returned object from fetching %d: '%s'"% (uid, dbg_output)) return body def getmessagelabels(self, uid): if 'labels' in self.messagelist[uid]: return self.messagelist[uid]['labels'] else: return set() # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0} # TODO: merge this code with the parent's cachemessagelist: # TODO: they have too much common logics. def cachemessagelist(self, min_date=None, min_uid=None): if not self.synclabels: return super(GmailFolder, self).cachemessagelist( min_date=min_date, min_uid=min_uid) self.dropmessagelistcache() self.ui.collectingdata(None, self) imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch( imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. # # NB: msgsToFetch are sequential numbers, not UID's res_type, response = imapobj.fetch("'%s'"% msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': six.reraise(OfflineImapError, OfflineImapError( "FETCHING UIDs in folder [%s]%s failed. "% (self.getrepository(), self) + "Server responded '[%s] %s'"% (res_type, response), OfflineImapError.ERROR.FOLDER), exc_info()[2]) finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] # e.g.: {'X-GM-LABELS': '("Webserver (RW.net)" "\\Inbox" GInbox)', 'FLAGS': '(\\Seen)', 'UID': '275440'} options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = int(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) # e.g.: '("Webserver (RW.net)" "\\Inbox" GInbox)' m = re.search('^[(](.*)[)]', options['X-GM-LABELS']) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime} def savemessage(self, uid, content, flags, rtime): """Save the message on the Server This backend always assigns a new uid, so the uid arg is ignored. This function will update the self.messagelist dict to contain the new message after sucessfully saving it, including labels. See folder/Base for details. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode. :param rtime: A timestamp to be used as the mail date :returns: the UID of the new message as assigned by the server. If the message is saved, but it's UID can not be found, it will return 0. If the message can't be written (folder is read-only for example) it will return -1.""" if not self.synclabels: return super(GmailFolder, self).savemessage(uid, content, flags, rtime) labels = set() for hstr in self.getmessageheaderlist(content, self.labelsheader): labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) self.savemessagelabels(ret, labels) return ret def _messagelabels_aux(self, arg, uidlist, labels): """Common code to savemessagelabels and addmessagelabels""" labels = labels - self.ignorelabels uidlist = [uid for uid in uidlist if uid > 0] if len(uidlist) > 0: imapobj = self.imapserver.acquireconnection() try: labels_str = '(' + ' '.join([imaputil.quote(lb) for lb in labels]) + ')' # Coalesce uid's into ranges uid_str = imaputil.uid_sequence(uidlist) result = self._store_to_imap(imapobj, uid_str, arg, labels_str) except imapobj.readonly: self.ui.labelstoreadonly(self, uidlist, labels) return None finally: self.imapserver.releaseconnection(imapobj) if result: retlabels = imaputil.flags2hash(imaputil.imapsplit(result)[1])['X-GM-LABELS'] retlabels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(retlabels)]) return retlabels return None def savemessagelabels(self, uid, labels): """Change a message's labels to `labels`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" if uid in self.messagelist and 'labels' in self.messagelist[uid]: oldlabels = self.messagelist[uid]['labels'] else: oldlabels = set() labels = labels - self.ignorelabels newlabels = labels | (oldlabels & self.ignorelabels) if oldlabels != newlabels: result = self._messagelabels_aux('X-GM-LABELS', [uid], newlabels) if result: self.messagelist[uid]['labels'] = newlabels else: self.messagelist[uid]['labels'] = oldlabels def addmessageslabels(self, uidlist, labels): """Add `labels` to all messages in uidlist. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" labels = labels - self.ignorelabels result = self._messagelabels_aux('+X-GM-LABELS', uidlist, labels) if result: for uid in uidlist: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels def deletemessageslabels(self, uidlist, labels): """Delete `labels` from all messages in uidlist. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" labels = labels - self.ignorelabels result = self._messagelabels_aux('-X-GM-LABELS', uidlist, labels) if result: for uid in uidlist: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels def copymessageto(self, uid, dstfolder, statusfolder, register = 1): """Copies a message from self to dst if needed, updating the status Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. :param uid: uid of the message to be copied. :param dstfolder: A BaseFolder-derived instance :param statusfolder: A LocalStatusFolder instance :param register: whether we should register a new thread." :returns: Nothing on success, or raises an Exception.""" # Check if we are really copying realcopy = uid > 0 and not dstfolder.uidexists(uid) # first copy the message super(GmailFolder, self).copymessageto(uid, dstfolder, statusfolder, register) # sync labels and mtime now when the message is new (the embedded labels are up to date) # otherwise we may be spending time for nothing, as they will get updated on a later pass. if realcopy and self.synclabels: try: mtime = dstfolder.getmessagemtime(uid) labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=mtime) # dstfolder is not GmailMaildir. except NotImplementedError: return def syncmessagesto_labels(self, dstfolder, statusfolder): """Pass 4: Label Synchronization (Gmail only) Compare label mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the labels change to both dstfolder and statusfolder. This function checks and protects us from action in dryrun mode. """ # This applies the labels message by message, as this makes more sense for a # Maildir target. If applied with an other Gmail IMAP target it would not be # the fastest thing in the world though... uidlist = [] # filter the uids (fast) try: for uid in self.getmessageuidlist(): # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break # Ignore messages with negative UIDs missed by pass 1 and # don't do anything if the message has been deleted remotely if uid < 0 or not dstfolder.uidexists(uid): continue selflabels = self.getmessagelabels(uid) - self.ignorelabels if statusfolder.uidexists(uid): statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels else: statuslabels = set() if selflabels != statuslabels: uidlist.append(uid) # now sync labels (slow) mtimes = {} labels = {} for i, uid in enumerate(uidlist): # bail out on CTRL-C or SIGTERM if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break selflabels = self.getmessagelabels(uid) - self.ignorelabels if statusfolder.uidexists(uid): statuslabels = statusfolder.getmessagelabels(uid) - self.ignorelabels else: statuslabels = set() if selflabels != statuslabels: self.ui.settinglabels(uid, i+1, len(uidlist), sorted(selflabels), dstfolder) if self.repository.account.dryrun: continue #don't actually add in a dryrun dstfolder.savemessagelabels(uid, selflabels, ignorelabels = self.ignorelabels) mtime = dstfolder.getmessagemtime(uid) mtimes[uid] = mtime labels[uid] = selflabels # Update statusfolder in a single DB transaction. It is safe, as if something fails, # statusfolder will be updated on the next run. statusfolder.savemessageslabelsbulk(labels) statusfolder.savemessagesmtimebulk(mtimes) except NotImplementedError: self.ui.warn("Can't sync labels. You need to configure a local repository of type GmailMaildir") ================================================ FILE: offlineimap/folder/GmailMaildir.py ================================================ # Maildir folder support with labels # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import six from sys import exc_info from .Maildir import MaildirFolder import offlineimap.accounts from offlineimap import OfflineImapError from offlineimap import imaputil class GmailMaildirFolder(MaildirFolder): """Folder implementation to support adding labels to messages in a Maildir.""" def __init__(self, root, name, sep, repository): super(GmailMaildirFolder, self).__init__(root, name, sep, repository) # The header under which labels are stored. self.labelsheader = self.repository.account.getconf('labelsheader', 'X-Keywords') # Enables / disables label sync. self.synclabels = self.repository.account.getconfboolean('synclabels', 0) # If synclabels is enabled, add a 4th pass to sync labels. if self.synclabels: self.syncmessagesto_passes.append(self.syncmessagesto_labels) def quickchanged(self, statusfolder): """Returns True if the Maildir has changed. Checks uids, flags and mtimes""" if self._utime_from_header is True: raise Exception("GmailMaildir does not support quick mode" " when 'utime_from_header' is enabled.") self.cachemessagelist() # Folder has different uids than statusfolder => TRUE. if sorted(self.getmessageuidlist()) != \ sorted(statusfolder.getmessageuidlist()): return True # Check for flag changes, it's quick on a Maildir. for (uid, message) in self.getmessagelist().items(): if message['flags'] != statusfolder.getmessageflags(uid): return True # check for newer mtimes. it is also fast for (uid, message) in self.getmessagelist().items(): if message['mtime'] > statusfolder.getmessagemtime(uid): return True return False # Nope, nothing changed. # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'flags': set(), 'labels': set(), 'labels_cached': False, 'filename': '/no-dir/no-such-file/', 'mtime': 0} def cachemessagelist(self, min_date=None, min_uid=None): if self.ismessagelistempty(): self.messagelist = self._scanfolder(min_date=min_date, min_uid=min_uid) # Get mtimes if self.synclabels: for uid, msg in list(self.messagelist.items()): filepath = os.path.join(self.getfullname(), msg['filename']) msg['mtime'] = int(os.stat(filepath).st_mtime) def getmessagelabels(self, uid): # Labels are not cached in cachemessagelist because it is too slow. if not self.messagelist[uid]['labels_cached']: filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) if not os.path.exists(filepath): return set() file = open(filepath, 'rt') content = file.read() file.close() self.messagelist[uid]['labels'] = set() for hstr in self.getmessageheaderlist(content, self.labelsheader): self.messagelist[uid]['labels'].update( imaputil.labels_from_header(self.labelsheader, hstr)) self.messagelist[uid]['labels_cached'] = True return self.messagelist[uid]['labels'] def getmessagemtime(self, uid): if not 'mtime' in self.messagelist[uid]: return 0 else: return self.messagelist[uid]['mtime'] def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" if not self.synclabels: return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) labels = set() for hstr in self.getmessageheaderlist(content, self.labelsheader): labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) # Update the mtime and labels. filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) self.messagelist[uid]['mtime'] = int(os.stat(filepath).st_mtime) self.messagelist[uid]['labels'] = labels return ret def savemessagelabels(self, uid, labels, ignorelabels=set()): """Change a message's labels to `labels`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) file = open(filepath, 'rt') content = file.read() file.close() oldlabels = set() for hstr in self.getmessageheaderlist(content, self.labelsheader): oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) labels = labels - ignorelabels ignoredlabels = oldlabels & ignorelabels oldlabels = oldlabels - ignorelabels # Nothing to change. if labels == oldlabels: return # Change labels into content. labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels | ignoredlabels)) # First remove old labels header, and then add the new one. content = self.deletemessageheaders(content, self.labelsheader) content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) mtime = int(os.stat(filepath).st_mtime) # Write file with new labels to a unique file in tmp. messagename = self.new_message_filename(uid, set()) tmpname = self.save_to_tmp_file(messagename, content) tmppath = os.path.join(self.getfullname(), tmpname) # Move to actual location. try: os.rename(tmppath, filepath) except OSError as e: six.reraise(OfflineImapError, OfflineImapError("Can't rename file '%s' to '%s': %s"% (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER), exc_info()[2]) # If utime_from_header=true, we don't want to change the mtime. if self._utime_from_header and mtime: os.utime(filepath, (mtime, mtime)) # save the new mtime and labels self.messagelist[uid]['mtime'] = int(os.stat(filepath).st_mtime) self.messagelist[uid]['labels'] = labels def copymessageto(self, uid, dstfolder, statusfolder, register=1): """Copies a message from self to dst if needed, updating the status Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode. :param uid: uid of the message to be copied. :param dstfolder: A BaseFolder-derived instance :param statusfolder: A LocalStatusFolder instance :param register: whether we should register a new thread." :returns: Nothing on success, or raises an Exception.""" # Check if we are really copying. realcopy = uid > 0 and not dstfolder.uidexists(uid) # First copy the message. super(GmailMaildirFolder, self).copymessageto(uid, dstfolder, statusfolder, register) # Sync labels and mtime now when the message is new (the embedded labels # are up to date, and have already propagated to the remote server. For # message which already existed on the remote, this is useless, as later # the labels may get updated. if realcopy and self.synclabels: try: labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=self.getmessagemtime(uid)) # dstfolder is not GmailMaildir. except NotImplementedError: return def syncmessagesto_labels(self, dstfolder, statusfolder): """Pass 4: Label Synchronization (Gmail only) Compare label mismatches in self with those in statusfolder. If msg has a valid UID and exists on dstfolder (has not e.g. been deleted there), sync the labels change to both dstfolder and statusfolder. Also skips messages whose mtime remains the same as statusfolder, as the contents have not changed. This function checks and protects us from action in ryrun mode. """ # For each label, we store a list of uids to which it should be # added. Then, we can call addmessageslabels() to apply them in # bulk, rather than one call per message. addlabellist = {} dellabellist = {} uidlist = [] try: # Filter uids (fast). for uid in self.getmessageuidlist(): # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break # Ignore messages with negative UIDs missed by pass 1 and # don't do anything if the message has been deleted remotely. if uid < 0 or not dstfolder.uidexists(uid): continue selfmtime = self.getmessagemtime(uid) if statusfolder.uidexists(uid): statusmtime = statusfolder.getmessagemtime(uid) else: statusmtime = 0 if selfmtime > statusmtime: uidlist.append(uid) self.ui.collectingdata(uidlist, self) # This can be slow if there is a lot of modified files. for uid in uidlist: # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break selflabels = self.getmessagelabels(uid) if statusfolder.uidexists(uid): statuslabels = statusfolder.getmessagelabels(uid) else: statuslabels = set() addlabels = selflabels - statuslabels dellabels = statuslabels - selflabels for lb in addlabels: if not lb in addlabellist: addlabellist[lb] = [] addlabellist[lb].append(uid) for lb in dellabels: if not lb in dellabellist: dellabellist[lb] = [] dellabellist[lb].append(uid) for lb, uids in addlabellist.items(): # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break self.ui.addinglabels(uids, lb, dstfolder) if self.repository.account.dryrun: continue # Don't actually add in a dryrun. dstfolder.addmessageslabels(uids, set([lb])) statusfolder.addmessageslabels(uids, set([lb])) for lb, uids in dellabellist.items(): # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break self.ui.deletinglabels(uids, lb, dstfolder) if self.repository.account.dryrun: continue # Don't actually remove in a dryrun. dstfolder.deletemessageslabels(uids, set([lb])) statusfolder.deletemessageslabels(uids, set([lb])) # Update mtimes on StatusFolder. It is done last to be safe. If # something els fails and the mtime is not updated, the labels will # still be synced next time. mtimes = {} for uid in uidlist: # Bail out on CTRL-C or SIGTERM. if offlineimap.accounts.Account.abort_NOW_signal.is_set(): break if self.repository.account.dryrun: continue # Don't actually update statusfolder. filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) mtimes[uid] = int(os.stat(filepath).st_mtime) # Finally, update statusfolder in a single DB transaction. statusfolder.savemessagesmtimebulk(mtimes) except NotImplementedError: self.ui.warn("Can't sync labels. You need to configure a remote " "repository of type Gmail.") ================================================ FILE: offlineimap/folder/IMAP.py ================================================ # IMAP folder support # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import random import binascii import re import time from sys import exc_info import six from .Base import BaseFolder from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError from offlineimap import globals from offlineimap.virtual_imaplib2 import MonthNames # Globals CRLF = '\r\n' MSGCOPY_NAMESPACE = 'MSGCOPY_' # NB: message returned from getmessage() will have '\n' all over the place, # NB: there will be no CRLFs. Just before the sending stage of savemessage() # NB: '\n' will be transformed back to CRLF. So, for the most parts of the # NB: code the stored content will be clean of CRLF and one can rely that # NB: line endings will be pure '\n'. class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository, decode=True): # decode the folder name from IMAP4_utf_7 to utf_8 if # - utf8foldernames is enabled for the *account* # - the decode argument is given # (default True is used when the folder name is the result of # querying the IMAP server, while False is used when creating # a folder object from a locally available utf_8 name) # In any case the given name is first dequoted. name = imaputil.dequote(name) if decode and repository.account.utf_8_support: name = imaputil.IMAP_utf8(name) self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) if repository.getdecodefoldernames(): self.visiblename = imaputil.decode_mailbox_name(self.visiblename) self.idle_mode = False self.expunge = repository.getexpunge() self.root = None # imapserver.root self.imapserver = imapserver self.randomgenerator = random.Random() # self.ui is set in BaseFolder. self.imap_query = ['BODY.PEEK[]'] # number of times to retry fetching messages self.retrycount = self.repository.getconfint('retrycount', 2) fh_conf = self.repository.account.getconf('filterheaders', '') self.filterheaders = [h for h in re.split(r'\s*,\s*', fh_conf) if h] # self.copy_ignoreUIDs is used by BaseFolder. self.copy_ignoreUIDs = repository.get_copy_ignore_UIDs( self.getvisiblename()) if self.repository.getidlefolders(): self.idle_mode = True def __selectro(self, imapobj, force=False): """Select this folder when we do not need write access. Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is selected. .. todo: Still valid? Needs verification :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: imapobj.select(self.getfullIMAPname(), force=force) except imapobj.readonly: imapobj.select(self.getfullIMAPname(), readonly=True, force=force) def getfullIMAPname(self): name = self.getfullname() if self.repository.account.utf_8_support: name = imaputil.utf8_IMAP(name) return name # Interface from BaseFolder def suggeststhreads(self): singlethreadperfolder_default = False if self.idle_mode is True: singlethreadperfolder_default = True onethread = self.config.getdefaultboolean( "Repository %s"% self.repository.getname(), "singlethreadperfolder", singlethreadperfolder_default) if onethread is True: return False return not globals.options.singlethreading # Interface from BaseFolder def waitforthread(self): self.imapserver.connectionwait() def getmaxage(self): if self.config.getdefault("Account %s"% self.accountname, "maxage", None): six.reraise(OfflineImapError, OfflineImapError( "maxage is not supported on IMAP-IMAP sync", OfflineImapError.ERROR.REPO), exc_info()[2]) # Interface from BaseFolder def getinstancelimitnamespace(self): return MSGCOPY_NAMESPACE + self.repository.getname() # Interface from BaseFolder def get_uidvalidity(self): """Retrieve the current connections UIDVALIDITY value UIDVALIDITY value will be cached on the first call. :returns: The UIDVALIDITY as (long) number.""" if hasattr(self, '_uidvalidity'): # Use cached value if existing. return self._uidvalidity imapobj = self.imapserver.acquireconnection() try: # SELECT (if not already done) and get current UIDVALIDITY. self.__selectro(imapobj) typ, uidval = imapobj.response('UIDVALIDITY') assert uidval != [None] and uidval != None, \ "response('UIDVALIDITY') returned [None]!" self._uidvalidity = int(uidval[-1]) return self._uidvalidity finally: self.imapserver.releaseconnection(imapobj) # Interface from BaseFolder def quickchanged(self, statusfolder): # An IMAP folder has definitely changed if the number of # messages or the UID of the last message have changed. Otherwise # only flag changes could have occurred. retry = True # Should we attempt another round or exit? while retry: retry = False imapobj = self.imapserver.acquireconnection() try: # Select folder and get number of messages. restype, imapdata = imapobj.select(self.getfullIMAPname(), True, True) self.imapserver.releaseconnection(imapobj) except OfflineImapError as e: # Retry on dropped connections, raise otherwise. self.imapserver.releaseconnection(imapobj, True) if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: retry = True else: raise except: # Cleanup and raise on all other errors. self.imapserver.releaseconnection(imapobj, True) raise # 1. Some mail servers do not return an EXISTS response # if the folder is empty. 2. ZIMBRA servers can return # multiple EXISTS replies in the form 500, 1000, 1500, # 1623 so check for potentially multiple replies. if imapdata == [None]: return True maxmsgid = 0 for msgid in imapdata: maxmsgid = max(int(msgid), maxmsgid) # Different number of messages than last time? if maxmsgid != statusfolder.getmessagecount(): return True return False def _msgs_to_fetch(self, imapobj, min_date=None, min_uid=None): """Determines sequence numbers of messages to be fetched. Message sequence numbers (MSNs) are more easily compacted into ranges which makes transactions slightly faster. Arguments: - imapobj: instance of IMAPlib - min_date (optional): a time_struct; only fetch messages newer than this - min_uid (optional): only fetch messages with UID >= min_uid This function should be called with at MOST one of min_date OR min_uid set but not BOTH. Returns: range(s) for messages or None if no messages are to be fetched.""" def search(search_conditions): """Actually request the server with the specified conditions. Returns: range(s) for messages or None if no messages are to be fetched.""" try: res_type, res_data = imapobj.search(None, search_conditions) if res_type != 'OK': raise OfflineImapError("SEARCH in folder [%s]%s failed. " "Search string was '%s'. Server responded '[%s] %s'"% ( self.getrepository(), self, search_cond, res_type, res_data), OfflineImapError.ERROR.FOLDER) except Exception as e: raise OfflineImapError("SEARCH in folder [%s]%s failed. " "Search string was '%s'. Error: %s"% ( self.getrepository(), self, search_cond, str(e)), OfflineImapError.ERROR.FOLDER) # Davmail returns list instead of list of one element string. # On first run the first element is empty. if ' ' in res_data[0] or res_data[0] == '': res_data = res_data[0].split() # Some servers are broken. if 0 in res_data: self.ui.warn("server returned UID with 0; ignoring.") res_data.remove(0) return res_data res_type, imapdata = imapobj.select(self.getfullIMAPname(), True, True) if imapdata == [None] or imapdata[0] == '0': # Empty folder, no need to populate message list. return None conditions = [] # 1. min_uid condition. if min_uid != None: conditions.append("UID %d:*"% min_uid) # 2. date condition. elif min_date != None: # Find out what the oldest message is that we should look at. conditions.append("SINCE %02d-%s-%d"% ( min_date[2], MonthNames[min_date[1]], min_date[0])) # 3. maxsize condition. maxsize = self.getmaxsize() if maxsize != None: conditions.append("SMALLER %d"% maxsize) if len(conditions) >= 1: # Build SEARCH command. search_cond = "(%s)"% ' '.join(conditions) search_result = search(search_cond) return imaputil.uid_sequence(search_result) # By default consider all messages in this folder. return '1:*' # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'uid': uid, 'flags': set(), 'time': 0} # Interface from BaseFolder def cachemessagelist(self, min_date=None, min_uid=None): self.ui.loadmessagelist(self.repository, self) self.dropmessagelistcache() imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch( imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync. # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. fetch_msg = "'%s'"% msgsToFetch self.ui.debug('imap', "calling imaplib2 fetch command: %s %s"% (fetch_msg, '(FLAGS UID INTERNALDATE)')) res_type, response = imapobj.fetch( fetch_msg, '(FLAGS UID INTERNALDATE)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " "Server responded '[%s] %s'"% (self.getrepository(), self, res_type, response), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # Looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg. # Discard initial message number. if messagestr is None: continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if 'UID' not in options: self.ui.warn('No UID in message with options %s'% str(options), minor=1) else: uid = int(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) keywords = imaputil.flagsimap2keywords(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'keywords': keywords} self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) # Interface from BaseFolder def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). After this function all CRLFs will be transformed to '\n'. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ data = self._fetch_from_imap(str(uid), self.retrycount) # Data looks now e.g. # [('320 (UID 17061 BODY[] {2565}','msgbody....')] # We only asked for one message, and that msg is in data[0]. msbody is # in [0][1]. data = data[0][1].replace(CRLF, "\n") if len(data) > 200: dbg_output = "%s...%s"% (str(data)[:150], str(data)[-50:]) else: dbg_output = data self.ui.debug('imap', "Returned object from fetching %d: '%s'"% (uid, dbg_output)) return data # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] # Interface from BaseFolder def getmessagekeywords(self, uid): return self.messagelist[uid]['keywords'] def __generate_randomheader(self, content): """Returns a unique X-OfflineIMAP header Generate an 'X-OfflineIMAP' mail header which contains a random unique value (which is based on the mail content, and a random number). This header allows us to fetch a mail after APPENDing it to an IMAP server and thus find out the UID that the server assigned it. :returns: (headername, headervalue) tuple, consisting of strings headername == 'X-OfflineIMAP' and headervalue will be a random string """ headername = 'X-OfflineIMAP' # We need a random component too. If we ever upload the same # mail twice (e.g. in different folders), we would still need to # get the UID for the correct one. As we won't have too many # mails with identical content, the randomness requirements are # not extremly critial though. # Compute unsigned crc32 of 'content' as unique hash. # NB: crc32 returns unsigned only starting with python 3.0. headervalue = str( binascii.crc32(content) & 0xffffffff ) + '-' headervalue += str(self.randomgenerator.randint(0,9999999999)) return (headername, headervalue) def __savemessage_searchforheader(self, imapobj, headername, headervalue): self.ui.debug('imap', '__savemessage_searchforheader called for %s: %s'% (headername, headervalue)) # Now find the UID it got. headervalue = imapobj._quote(headervalue) try: matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0] except imapobj.error as err: # IMAP server doesn't implement search or had a problem. self.ui.debug('imap', "__savemessage_searchforheader: got IMAP " "error '%s' while attempting to UID SEARCH for message with " "header %s"% (err, headername)) return 0 self.ui.debug('imap', "__savemessage_searchforheader got initial " "matchinguids: " + repr(matchinguids)) if matchinguids == '': self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH " "for message with header %s yielded no results"% headername) return 0 matchinguids = matchinguids.split(' ') self.ui.debug('imap', '__savemessage_searchforheader: matchinguids now ' + repr(matchinguids)) if len(matchinguids) != 1 or matchinguids[0] is None: raise OfflineImapError( "While attempting to find UID for message with " "header %s, got wrong-sized matchinguids of %s"% (headername, str(matchinguids)), OfflineImapError.ERROR.MESSAGE ) return int(matchinguids[0]) def __savemessage_fetchheaders(self, imapobj, headername, headervalue): """ We fetch all new mail headers and search for the right X-OfflineImap line by hand. The response from the server has form: ( 'OK', [ ( '185 (RFC822.HEADER {1789}', '... mail headers ...' ), ' UID 2444)', ( '186 (RFC822.HEADER {1789}', '... 2nd mail headers ...' ), ' UID 2445)' ] ) We need to locate the UID just after mail headers containing our X-OfflineIMAP line. Returns UID when found, 0 when not found.""" self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \ (headername, headervalue)) # Run "fetch X:* rfc822.header". # Since we stored the mail we are looking for just recently, it would # not be optimal to fetch all messages. So we'll find highest message # UID in our local messagelist and search from there (exactly from # UID+1). That works because UIDs are guaranteed to be unique and # ascending. if self.getmessagelist(): start = 1 + max(self.getmessagelist().keys()) else: # Folder was empty - start from 1. start = 1 # Imaplib quotes all parameters of a string type. That must not happen # with the range X:*. So we use bytearray to stop imaplib from getting # in our way. result = imapobj.uid('FETCH', bytearray('%d:*'% start), 'rfc822.header') if result[0] != 'OK': raise OfflineImapError('Error fetching mail headers: %s'% '. '.join(result[1]), OfflineImapError.ERROR.MESSAGE) # result is like: # [ # ('185 (RFC822.HEADER {1789}', '... mail headers ...'), ' UID 2444)', # ('186 (RFC822.HEADER {1789}', '... 2nd mail headers ...'), ' UID 2445)' # ] result = result[1] found = None # item is like: # ('185 (RFC822.HEADER {1789}', '... mail headers ...'), ' UID 2444)' for item in result: if found is None and type(item) == tuple: # Walk just tuples. if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)"% (headername, headervalue), item[1], flags=re.IGNORECASE): found = item[0] elif found is not None: if type(item) == type(""): uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE) if uid: return int(uid.group(1)) else: # This parsing is for Davmail. # https://github.com/OfflineIMAP/offlineimap/issues/479 # item is like: # ')' # and item[0] stored in "found" is like: # '1694 (UID 1694 RFC822.HEADER {1294}' uid = re.search("\d+\s+\(UID\s+(\d+)", found, flags=re.IGNORECASE) if uid: return int(uid.group(1)) self.ui.warn("Can't parse FETCH response, can't find UID in %s"% item ) self.ui.debug('imap', "Got: %s"% repr(result)) else: self.ui.warn("Can't parse FETCH response, we awaited string: %s"% repr(item) ) return 0 def __getmessageinternaldate(self, content, rtime=None): """Parses mail and returns an INTERNALDATE string It will use information in the following order, falling back as an attempt fails: - rtime parameter - Date header of email We return None, if we couldn't find a valid date. In this case the IMAP server will use the server local time when appening (per RFC). Note, that imaplib's Time2Internaldate is inherently broken as it returns localized date strings which are invalid for IMAP servers. However, that function is called for *every* append() internally. So we need to either pass in `None` or the correct string (in which case Time2Internaldate() will do nothing) to append(). The output of this function is designed to work as input to the imapobj.append() function. TODO: We should probably be returning a bytearray rather than a string here, because the IMAP server will expect plain ASCII. However, imaplib.Time2INternaldate currently returns a string so we go with the same for now. :param rtime: epoch timestamp to be used rather than analyzing the email. :returns: string in the form of "DD-Mmm-YYYY HH:MM:SS +HHMM" (including double quotes) or `None` in case of failure (which is fine as value for append).""" if rtime is None: rtime = emailutil.get_message_date(content) if rtime == None: return None datetuple = time.localtime(rtime) try: # Check for invalid dates. if datetuple[0] < 1981: raise ValueError # Check for invalid dates. datetuple_check = time.localtime(time.mktime(datetuple)) if datetuple[:2] != datetuple_check[:2]: raise ValueError except (ValueError, OverflowError): # Argh, sometimes it's a valid format but year is 0102 # or something. Argh. It seems that Time2Internaldate # will rause a ValueError if the year is 0102 but not 1902, # but some IMAP servers nonetheless choke on 1902. self.ui.debug('imap', "Message with invalid date %s. " "Server will use local time."% datetuple) return None # Produce a string representation of datetuple that works as # INTERNALDATE. num2mon = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'} # tm_isdst coming from email.parsedate is not usable, we still use it # here, mhh. if datetuple.tm_isdst == 1: zone = -time.altzone else: zone = -time.timezone offset_h, offset_m = divmod(zone//60, 60) internaldate = '"%02d-%s-%04d %02d:%02d:%02d %+03d%02d"'% \ (datetuple.tm_mday, num2mon[datetuple.tm_mon], datetuple.tm_year, \ datetuple.tm_hour, datetuple.tm_min, datetuple.tm_sec, offset_h, offset_m) return internaldate # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Save the message on the Server This backend always assigns a new uid, so the uid arg is ignored. This function will update the self.messagelist dict to contain the new message after sucessfully saving it. See folder/Base for details. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode. :param rtime: A timestamp to be used as the mail date :returns: the UID of the new message as assigned by the server. If the message is saved, but it's UID can not be found, it will return 0. If the message can't be written (folder is read-only for example) it will return -1.""" self.ui.savemessage('imap', uid, flags, self) # Already have it, just save modified flags. if uid > 0 and self.uidexists(uid): self.savemessageflags(uid, flags) return uid content = self.deletemessageheaders(content, self.filterheaders) # Use proper CRLF all over the message. content = re.sub("(? 200: dbg_output = "%s...%s"% (content[:150], content[-50:]) else: dbg_output = content self.ui.debug('imap', "savemessage: date: %s, content: '%s'"% (date, dbg_output)) try: # Select folder for append and make the box READ-WRITE. imapobj.select(self.getfullIMAPname()) except imapobj.readonly: # readonly exception. Return original uid to notify that # we did not save the message. (see savemessage in Base.py) self.ui.msgtoreadonly(self, uid, content, flags) return uid # Do the APPEND. try: (typ, dat) = imapobj.append(self.getfullIMAPname(), imaputil.flagsmaildir2imap(flags), date, content) # This should only catch 'NO' responses since append() # will raise an exception for 'BAD' responses: if typ != 'OK': # For example, Groupwise IMAP server can return something like: # # NO APPEND The 1500 MB storage limit has been exceeded. # # In this case, we should immediately abort the repository sync # and continue with the next account. msg = \ "Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \ "Server responded: %s %s\n"% \ (msg_id, self, self.getrepository(), typ, dat) raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) retry_left = 0 # Mark as success. except imapobj.abort as e: # Connection has been reset, release connection and retry. retry_left -= 1 self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() if not retry_left: six.reraise(OfflineImapError, OfflineImapError("Saving msg (%s) in folder '%s', " "repository '%s' failed (abort). Server responded: %s\n" "Message content was: %s"% (msg_id, self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE), exc_info()[2]) # XXX: is this still needed? self.ui.error(e, exc_info()[2]) except imapobj.error as e: # APPEND failed # If the server responds with 'BAD', append() # raise()s directly. So we catch that too. # drop conn, it might be bad. self.imapserver.releaseconnection(imapobj, True) imapobj = None six.reraise(OfflineImapError, OfflineImapError("Saving msg (%s) folder '%s', repo '%s'" "failed (error). Server responded: %s\nMessage content was: " "%s"% (msg_id, self, self.getrepository(), str(e), dbg_output), OfflineImapError.ERROR.MESSAGE), exc_info()[2]) # Checkpoint. Let it write out stuff, etc. Eg searches for # just uploaded messages won't work if we don't do this. (typ,dat) = imapobj.check() assert(typ == 'OK') # Get the new UID, do we use UIDPLUS? if use_uidplus: # Get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with # 38505 bein folder UIDvalidity and 3955 the new UID. # note: we would want to use .response() here but that # often seems to return [None], even though we have # data. TODO resp = imapobj._get_untagged_response('APPENDUID') if resp == [None] or resp is None: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message. Got: %s."% str(resp)) return 0 try: uid = int(resp[-1].split(' ')[1]) except ValueError as e: uid = 0 # Definetly not what we should have. except Exception as e: raise OfflineImapError("Unexpected response: %s"% str(resp), OfflineImapError.ERROR.MESSAGE) if uid == 0: self.ui.warn("savemessage: Server supports UIDPLUS, but" " we got no usable UID back. APPENDUID reponse was " "'%s'"% str(resp)) else: try: # We don't use UIDPLUS. uid = self.__savemessage_searchforheader(imapobj, headername, headervalue) # See docs for savemessage in Base.py for explanation # of this and other return values. if uid == 0: self.ui.debug('imap', 'savemessage: attempt to get new UID ' 'UID failed. Search headers manually.') uid = self.__savemessage_fetchheaders(imapobj, headername, headervalue) self.ui.warn("savemessage: Searching mails for new " "Message-ID failed. Could not determine new UID " "on %s."% self.getname()) # Something wrong happened while trying to get the UID. Explain # the error might be about the 'get UID' process not necesseraly # the APPEND. except Exception: self.ui.warn("%s: could not determine the UID while we got " "no error while appending the email with '%s: %s'"% (self.getname(), headername, headervalue) ) raise finally: if imapobj: self.imapserver.releaseconnection(imapobj) if uid: # Avoid UID FETCH 0 crash happening later on. self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags self.ui.debug('imap', 'savemessage: returning new UID %d'% uid) return uid def _fetch_from_imap(self, uids, retry_num=1): """Fetches data from IMAP server. Arguments: - uids: message UIDS - retry_num: number of retries to make Returns: data obtained by this query.""" imapobj = self.imapserver.acquireconnection() try: query = "(%s)"% (" ".join(self.imap_query)) fails_left = retry_num # Retry on dropped connection. while fails_left: try: imapobj.select(self.getfullIMAPname(), readonly=True) res_type, data = imapobj.uid('fetch', uids, query) break except imapobj.abort as e: fails_left -= 1 # self.ui.error() will show the original traceback. if fails_left <= 0: message = ("%s, while fetching msg %r in folder %r." " Max retry reached (%d)"% (e, uids, self.name, retry_num)) severity = OfflineImapError.ERROR.MESSAGE raise OfflineImapError(message, OfflineImapError.ERROR.MESSAGE) self.ui.error("%s. While fetching msg %r in folder %r." " Query: %s Retrying (%d/%d)"% ( e, uids, self.name, query, retry_num - fails_left, retry_num ) ) # Release dropped connection, and get a new one. self.imapserver.releaseconnection(imapobj, True) imapobj = self.imapserver.acquireconnection() finally: # The imapobj here might be different than the one created before # the ``try`` clause. So please avoid transforming this to a nice # ``with`` without taking this into account. self.imapserver.releaseconnection(imapobj) # Ensure to not consider unsolicited FETCH responses caused by flag # changes from concurrent connections. These appear as strings in # 'data' (the BODY response appears as a tuple). This should leave # exactly one response. if res_type == 'OK': data = [res for res in data if not isinstance(res, str)] # Could not fetch message. Note: it is allowed by rfc3501 to return any # data for the UID FETCH command. if data == [None] or res_type != 'OK' or len(data) != 1: severity = OfflineImapError.ERROR.MESSAGE reason = "IMAP server '%s' failed to fetch messages UID '%s'."\ " Server responded: %s %s"% (self.getrepository(), uids, res_type, data) if data == [None] or len(data) < 1: # IMAP server did not find a message with this UID. reason = "IMAP server '%s' does not have a message "\ "with UID '%s'"% (self.getrepository(), uids) raise OfflineImapError(reason, severity) return data def _store_to_imap(self, imapobj, uid, field, data): """Stores data to IMAP server Arguments: - imapobj: instance of IMAPlib to use - uid: message UID - field: field name to be stored/updated - data: field contents """ imapobj.select(self.getfullIMAPname()) res_type, retdata = imapobj.uid('store', uid, field, data) if res_type != 'OK': severity = OfflineImapError.ERROR.MESSAGE reason = "IMAP server '%s' failed to store %s for message UID '%d'."\ "Server responded: %s %s"% ( self.getrepository(), field, uid, res_type, retdata) raise OfflineImapError(reason, severity) return retdata[0] # Interface from BaseFolder def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: result = self._store_to_imap(imapobj, str(uid), 'FLAGS', imaputil.flagsmaildir2imap(flags)) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return finally: self.imapserver.releaseconnection(imapobj) if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags) # Interface from BaseFolder def addmessageflags(self, uid, flags): self.addmessagesflags([uid], flags) def __addmessagesflags_noconvert(self, uidlist, flags): self.__processmessagesflags('+', uidlist, flags) # Interface from BaseFolder def addmessagesflags(self, uidlist, flags): """This is here for the sake of UIDMaps.py -- deletemessages must add flags and get a converted UID, and if we don't have noconvert, then UIDMaps will try to convert it twice.""" self.__addmessagesflags_noconvert(uidlist, flags) # Interface from BaseFolder def deletemessageflags(self, uid, flags): self.deletemessagesflags([uid], flags) # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): self.__processmessagesflags('-', uidlist, flags) def __processmessagesflags_real(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return response = imapobj.uid('store', imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) if response[0] != 'OK': raise OfflineImapError( 'Error with store: %s'% '. '.join(response[1]), OfflineImapError.ERROR.MESSAGE) response = response[1] finally: self.imapserver.releaseconnection(imapobj) # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. needupdate = list(uidlist) for result in response: if result is None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue flagstr = attributehash['FLAGS'] uid = int(attributehash['UID']) self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list. pass for uid in needupdate: if operation == '+': self.messagelist[uid]['flags'] |= flags elif operation == '-': self.messagelist[uid]['flags'] -= flags def __processmessagesflags(self, operation, uidlist, flags): # Hack for those IMAP servers with a limited line length. batch_size = 100 for i in range(0, len(uidlist), batch_size): self.__processmessagesflags_real(operation, uidlist[i:i + batch_size], flags) return # Interface from BaseFolder def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid If the backend supports it. IMAP does not and will throw errors.""" raise OfflineImapError('IMAP backend cannot change a messages UID from ' '%d to %d'% (uid, new_uid), OfflineImapError.ERROR.MESSAGE) # Interface from BaseFolder def deletemessage(self, uid): self.__deletemessages_noconvert([uid]) # Interface from BaseFolder def deletemessages(self, uidlist): self.__deletemessages_noconvert(uidlist) def __deletemessages_noconvert(self, uidlist): if not len(uidlist): return self.__addmessagesflags_noconvert(uidlist, set('T')) imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullIMAPname()) except imapobj.readonly: self.ui.deletereadonly(self, uidlist) return if self.expunge: assert(imapobj.expunge()[0] == 'OK') finally: self.imapserver.releaseconnection(imapobj) for uid in uidlist: del self.messagelist[uid] ================================================ FILE: offlineimap/folder/LocalStatus.py ================================================ # Local status cache virtual folder # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from sys import exc_info import os import threading import six from .Base import BaseFolder class LocalStatusFolder(BaseFolder): """LocalStatus backend implemented as a plain text file.""" cur_version = 2 magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT %d" def __init__(self, name, repository): self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) self.root = repository.root self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.savelock = threading.Lock() # Should we perform fsyncs as often as possible? self.doautosave = self.config.getdefaultboolean( "general", "fsync", False) # Interface from BaseFolder def storesmessages(self): return 0 def isnewfolder(self): return not os.path.exists(self.filename) # Interface from BaseFolder def getfullname(self): return self.filename # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} def readstatus_v1(self, fp): """Read status folder in format version 1. Arguments: - fp: I/O object that points to the opened database file. """ for line in fp: line = line.strip() try: uid, flags = line.split(':') uid = int(uid) flags = set(flags) except ValueError as e: errstr = ("Corrupt line '%s' in cache file '%s'"% (line, self.filename)) self.ui.warn(errstr) six.reraise(ValueError, ValueError(errstr), exc_info()[2]) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags def readstatus(self, fp): """Read status file in the current format. Arguments: - fp: I/O object that points to the opened database file. """ for line in fp: line = line.strip() try: uid, flags, mtime, labels = line.split('|') uid = int(uid) flags = set(flags) mtime = int(mtime) labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) except ValueError as e: errstr = "Corrupt line '%s' in cache file '%s'"% \ (line, self.filename) self.ui.warn(errstr) six.reraise(ValueError, ValueError(errstr), exc_info()[2]) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['mtime'] = mtime self.messagelist[uid]['labels'] = labels # Interface from BaseFolder def cachemessagelist(self): if self.isnewfolder(): self.dropmessagelistcache() return # Loop as many times as version, and update format. for i in range(1, self.cur_version + 1): self.dropmessagelistcache() cachefd = open(self.filename, "rt") line = cachefd.readline().strip() # Format is up to date. break. if line == (self.magicline % self.cur_version): break # Convert from format v1. elif line == (self.magicline % 1): self.ui._msg('Upgrading LocalStatus cache from version 1 ' 'to version 2 for %s:%s'% (self.repository, self)) self.readstatus_v1(cachefd) cachefd.close() self.save() # NOTE: Add other format transitions here in the future. # elif line == (self.magicline % 2): # self.ui._msg(u'Upgrading LocalStatus cache from version 2' # 'to version 3 for %s:%s'% (self.repository, self)) # self.readstatus_v2(cache) # cache.close() # cache.save() # Something is wrong. else: errstr = "Unrecognized cache magicline in '%s'" % self.filename self.ui.warn(errstr) raise ValueError(errstr) if not line: # The status file is empty - should not have happened, # but somehow did. errstr = "Cache file '%s' is empty."% self.filename self.ui.warn(errstr) cachefd.close() return assert(line == (self.magicline % self.cur_version)) self.readstatus(cachefd) cachefd.close() def openfiles(self): pass # Closing files is done on a per-transaction basis. def closefiles(self): pass # Closing files is done on a per-transaction basis. def purge(self): """Remove any pre-existing database.""" try: os.unlink(self.filename) except OSError as e: self.ui.debug('', "could not remove file %s: %s"% (self.filename, e)) def save(self): """Save changed data to disk. For this backend it is the same as saveall.""" self.saveall() def saveall(self): """Saves the entire messagelist to disk.""" with self.savelock: cachefd = open(self.filename + ".tmp", "wt") cachefd.write((self.magicline % self.cur_version) + "\n") for msg in self.messagelist.values(): flags = ''.join(sorted(msg['flags'])) labels = ', '.join(sorted(msg['labels'])) cachefd.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) cachefd.flush() if self.doautosave: os.fsync(cachefd.fileno()) cachefd.close() os.rename(self.filename + ".tmp", self.filename) if self.doautosave: fd = os.open(os.path.dirname(self.filename), os.O_RDONLY) os.fsync(fd) os.close(fd) # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" if uid < 0: # We cannot assign a uid. return uid if self.uidexists(uid): # already have it self.savemessageflags(uid, flags) return uid self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['time'] = rtime self.messagelist[uid]['mtime'] = mtime self.messagelist[uid]['labels'] = labels self.save() return uid # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] # Interface from BaseFolder def getmessagetime(self, uid): return self.messagelist[uid]['time'] # Interface from BaseFolder def savemessageflags(self, uid, flags): self.messagelist[uid]['flags'] = flags self.save() def savemessagelabels(self, uid, labels, mtime=None): self.messagelist[uid]['labels'] = labels if mtime: self.messagelist[uid]['mtime'] = mtime self.save() def savemessageslabelsbulk(self, labels): """Saves labels from a dictionary in a single database operation.""" for uid, lb in labels.items(): self.messagelist[uid]['labels'] = lb self.save() def addmessageslabels(self, uids, labels): for uid in uids: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels self.save() def deletemessageslabels(self, uids, labels): for uid in uids: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels self.save() def getmessagelabels(self, uid): return self.messagelist[uid]['labels'] def savemessagesmtimebulk(self, mtimes): """Saves mtimes from the mtimes dictionary in a single database operation.""" for uid, mt in mtimes.items(): self.messagelist[uid]['mtime'] = mt self.save() def getmessagemtime(self, uid): return self.messagelist[uid]['mtime'] # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) # Interface from BaseFolder def deletemessages(self, uidlist): # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): return for uid in uidlist: del(self.messagelist[uid]) self.save() ================================================ FILE: offlineimap/folder/LocalStatusSQLite.py ================================================ # Local status cache virtual folder: SQLite backend # Copyright (C) 2009-2017 Stewart Smith and contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import sqlite3 as sqlite from sys import exc_info from threading import Lock import six from .Base import BaseFolder class DatabaseFileLock(object): """Lock at database file level.""" def __init__(self): self._lock = Lock() self._counter = 0 def __enter__(self): self._lock.acquire() def __exit__(self, typ, value, tb): self._lock.release() def registerNewUser(self): self._counter += 1 def removeOneUser(self): self._counter -= 1 def getLock(self): return self._lock def shouldClose(self): return self._counter < 1 class LocalStatusSQLiteFolder(BaseFolder): """LocalStatus backend implemented with an SQLite database As python-sqlite currently does not allow to access the same sqlite objects from various threads, we need to open get and close a db connection and cursor for all operations. This is a big disadvantage and we might want to investigate if we cannot hold an object open for a thread somehow.""" # Though. According to sqlite docs, you need to commit() before # the connection is closed or your changes will be lost! # get db connection which autocommits # connection = sqlite.connect(self.filename, isolation_level=None) # cursor = connection.cursor() # return connection, cursor # Current version of our db format. cur_version = 2 # Keep track on how many threads need access to the database. locks = {} # Key: filename, value: DatabaseFileLock instance. def __init__(self, name, repository): self.sep = '.' # Needs to be set before super().__init__(). super(LocalStatusSQLiteFolder, self).__init__(name, repository) self.root = repository.root self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self._newfolder = False # Flag if the folder is new. dirname = os.path.dirname(self.filename) if not os.path.exists(dirname): os.makedirs(dirname) if not os.path.isdir(dirname): raise UserWarning("SQLite database path '%s' is not a directory."% dirname) self.connection = None # The lock serialize the writing/open/close of database accross threads. if self.filename not in LocalStatusSQLiteFolder.locks: LocalStatusSQLiteFolder.locks[self.filename] = DatabaseFileLock() self._databaseFileLock = LocalStatusSQLiteFolder.locks[self.filename] self._in_transactions = 0 def __enter__(self): if not self.dofsync(): assert self.connection is not None self._in_transactions += 1 def __exit__(self, exc_type, exc_val, exc_tb): if not self.dofsync(): assert self._in_transactions > 0 self._in_transactions -= 1 if self._in_transactions < 1: self.connection.commit() def openfiles(self): # Make sure sqlite is in multithreading SERIALIZE mode. assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' with self._databaseFileLock.getLock(): # Try to establish connection, no need for threadsafety in __init__. try: self.connection = sqlite.connect(self.filename, check_same_thread=False) self._databaseFileLock.registerNewUser() except sqlite.OperationalError as e: # Operation had failed. six.reraise(UserWarning, UserWarning( "cannot open database file '%s': %s.\nYou might" " want to check the rights to that file and if " "it cleanly opens with the 'sqlite<3>' command"% (self.filename, e)), exc_info()[2]) # Test if db version is current enough and if db is readable. try: cursor = self.connection.execute( "SELECT value from metadata WHERE key='db_version'") except sqlite.DatabaseError: # db file missing or corrupt, recreate it. self.__create_db() else: # Fetch db version and upgrade if needed. version = int(cursor.fetchone()[0]) if version < LocalStatusSQLiteFolder.cur_version: self.__upgrade_db(version) def purge(self): """Remove any pre-existing database. Do not call in dry-run mode.""" try: os.unlink(self.filename) except OSError as e: self.ui.debug('', "could not remove file %s: %s"% (self.filename, e)) def storesmessages(self): return False def getfullname(self): return self.filename # Interface from LocalStatusFolder def isnewfolder(self): return self._newfolder def __sql_write(self, sql, args=None, executemany=False): """Execute some SQL, retrying if the db was locked. :param sql: the SQL string passed to execute() :param args: the variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See sqlite docs for possibilities. :param executemany: bool indicating whether we want to perform conn.executemany() or conn.execute(). :returns: None or raises an Exception.""" success = False while not success: try: with self._databaseFileLock.getLock(): if args is None: if executemany: self.connection.executemany(sql) else: self.connection.execute(sql) else: if executemany: self.connection.executemany(sql, args) else: self.connection.execute(sql, args) success = True if not self._in_transactions: self.connection.commit() except sqlite.OperationalError as e: if e.args[0] == 'cannot commit - no transaction is active': pass elif e.args[0] == 'database is locked': self.ui.debug('', "Locked sqlite database, retrying.") success = False else: raise def __upgrade_db(self, from_ver): """Upgrade the sqlite format from version 'from_ver' to current""" if self.connection is not None: self.connection.close() # Close old connections first. self.connection = sqlite.connect(self.filename, check_same_thread=False) # Upgrade from database version 1 to version 2 # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. if from_ver <= 1: self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s'% (self.repository, self)) self.connection.executescript("""ALTER TABLE status ADD mtime INTEGER DEFAULT 0; ALTER TABLE status ADD labels VARCHAR(256) DEFAULT ''; UPDATE metadata SET value='2' WHERE key='db_version'; """) self.connection.commit() # Future version upgrades come here... # if from_ver <= 2: ... #upgrade from 2 to 3 # if from_ver <= 3: ... #upgrade from 3 to 4 def __create_db(self): """Create a new db file. self.connection must point to the opened and valid SQlite database connection.""" self.ui._msg('Creating new Local Status db for %s:%s'% (self.repository, self)) self.connection.executescript(""" CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); INSERT INTO metadata VALUES('db_version', '2'); CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50), mtime INTEGER, labels VARCHAR(256)); """) self.connection.commit() self._newfolder = True # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0} # Interface from BaseFolder def cachemessagelist(self): self.dropmessagelistcache() cursor = self.connection.execute('SELECT id,flags,mtime,labels from status') for row in cursor: uid = row[0] self.messagelist[uid] = self.msglist_item_initializer(uid) flags = set(row[1]) try: labels = set([lb.strip() for lb in row[3].split(',') if len(lb.strip()) > 0]) except AttributeError: # FIXME: This except clause was introduced because row[3] from # database can be found of unexpected type NoneType. See # https://github.com/OfflineIMAP/offlineimap/issues/103 # # We are fixing the type here but this would require more # researches to find the true root cause. row[3] is expected to # be a (empty) string, not None. # # Also, since database might return None, we have to fix the # database, too. labels = set() self.messagelist[uid]['flags'] = flags self.messagelist[uid]['labels'] = labels self.messagelist[uid]['mtime'] = row[2] def closefiles(self): with self._databaseFileLock.getLock(): self._databaseFileLock.removeOneUser() if self._databaseFileLock.shouldClose(): try: self.connection.close() except: pass # Interface from LocalStatusFolder def save(self): pass # Noop. every transaction commits to database! def saveall(self): """Saves the entire messagelist to the database.""" with self._databaseFileLock.getLock(): data = [] for uid, msg in self.messagelist.items(): mtime = msg['mtime'] flags = ''.join(sorted(msg['flags'])) labels = ', '.join(sorted(msg['labels'])) data.append((uid, flags, mtime, labels)) self.__sql_write('INSERT OR REPLACE INTO status ' '(id,flags,mtime,labels) VALUES (?,?,?,?)', data, executemany=True) # Following some pure SQLite functions, where we chose to use # BaseFolder() methods instead. Doing those on the in-memory list is # quicker anyway. If our db becomes so big that we don't want to # maintain the in-memory list anymore, these might come in handy # in the future though. # #def uidexists(self,uid): # conn, cursor = self.get_cursor() # with conn: # cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid}) # return cursor.fetchone() # This would be the pure SQLite solution, use BaseFolder() method, # to avoid threading with sqlite... #def getmessageuidlist(self): # conn, cursor = self.get_cursor() # with conn: # cursor.execute('SELECT id from status') # r = [] # for row in cursor: # r.append(row[0]) # return r #def getmessagecount(self): # conn, cursor = self.get_cursor() # with conn: # cursor.execute('SELECT count(id) from status'); # return cursor.fetchone()[0] #def getmessageflags(self, uid): # conn, cursor = self.get_cursor() # with conn: # cursor.execute('SELECT flags FROM status WHERE id=:id', # {'id': uid}) # for row in cursor: # flags = [x for x in row[0]] # return flags # assert False,"getmessageflags() called on non-existing message" # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" if uid < 0: # We cannot assign a uid. return uid if self.uidexists(uid): # Already have it. self.savemessageflags(uid, flags) return uid self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} flags = ''.join(sorted(flags)) labels = ', '.join(sorted(labels)) try: self.__sql_write('INSERT INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)', (uid,flags,mtime,labels)) except Exception as e: six.reraise(UserWarning, UserWarning("%s while inserting UID %s"% (str(e), str(uid))), exc_info()[2]) return uid # Interface from BaseFolder def savemessageflags(self, uid, flags): assert self.uidexists(uid) self.messagelist[uid]['flags'] = flags flags = ''.join(sorted(flags)) self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) def getmessageflags(self, uid): return self.messagelist[uid]['flags'] def savemessagelabels(self, uid, labels, mtime=None): self.messagelist[uid]['labels'] = labels if mtime: self.messagelist[uid]['mtime'] = mtime labels = ', '.join(sorted(labels)) if mtime: self.__sql_write('UPDATE status SET labels=?, mtime=? WHERE id=?',(labels,mtime,uid)) else: self.__sql_write('UPDATE status SET labels=? WHERE id=?',(labels,uid)) def savemessageslabelsbulk(self, labels): """ Saves labels from a dictionary in a single database operation. """ data = [(', '.join(sorted(l)), uid) for uid, l in labels.items()] self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) for uid, l in labels.items(): self.messagelist[uid]['labels'] = l def addmessageslabels(self, uids, labels): data = [] for uid in uids: newlabels = self.messagelist[uid]['labels'] | labels data.append((', '.join(sorted(newlabels)), uid)) self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) for uid in uids: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels def deletemessageslabels(self, uids, labels): data = [] for uid in uids: newlabels = self.messagelist[uid]['labels'] - labels data.append((', '.join(sorted(newlabels)), uid)) self.__sql_write('UPDATE status SET labels=? WHERE id=?', data, executemany=True) for uid in uids: self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels def getmessagelabels(self, uid): return self.messagelist[uid]['labels'] def savemessagesmtimebulk(self, mtimes): """Saves mtimes from the mtimes dictionary in a single database operation.""" data = [(mt, uid) for uid, mt in mtimes.items()] self.__sql_write('UPDATE status SET mtime=? WHERE id=?', data, executemany=True) for uid, mt in mtimes.items(): self.messagelist[uid]['mtime'] = mt def getmessagemtime(self, uid): return self.messagelist[uid]['mtime'] # Interface from BaseFolder def deletemessage(self, uid): if not uid in self.messagelist: return self.__sql_write('DELETE FROM status WHERE id=?', (uid, )) del(self.messagelist[uid]) # Interface from BaseFolder def deletemessages(self, uidlist): """Delete list of UIDs from status cache This function uses sqlites executemany() function which is much faster than iterating through deletemessage() when we have many messages to delete.""" # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): return # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...] self.__sql_write('DELETE FROM status WHERE id=?', list(zip(uidlist, )), True) for uid in uidlist: del(self.messagelist[uid]) ================================================ FILE: offlineimap/folder/Maildir.py ================================================ # Maildir folder support # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import socket import time import re import os import six from sys import exc_info from threading import Lock try: from hashlib import md5 except ImportError: from md5 import md5 try: # python 2.6 has set() built in set except NameError: from sets import Set as set from offlineimap import OfflineImapError, emailutil from .Base import BaseFolder # Find the UID in a message filename re_uidmatch = re.compile(',U=(\d+)') # Find a numeric timestamp in a string (filename prefix) re_timestampmatch = re.compile('(\d+)') timehash = {} timelock = Lock() def _gettimeseq(date=None): global timehash, timelock timelock.acquire() try: if date is None: date = int(time.time()) if date in timehash: timehash[date] += 1 else: timehash[date] = 0 return (date, timehash[date]) finally: timelock.release() class MaildirFolder(BaseFolder): def __init__(self, root, name, sep, repository): self.sep = sep # needs to be set before super().__init__ super(MaildirFolder, self).__init__(name, repository) self.root = root # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" self.re_flagmatch = re.compile('%s2,(\w*)'% self.infosep) #self.ui is set in BaseFolder.init() # Everything up to the first comma or colon (or ! if Windows): self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') # folder's md, so we can match with recorded file md5 for validity. self._foldermd5 = md5(self.getvisiblename()).hexdigest() # Cache the full folder path, as we use getfullname() very often. self._fullname = os.path.join(self.getroot(), self.getname()) # Modification time from 'Date' header. utime_from_header_global = self.config.getdefaultboolean( "general", "utime_from_header", False) self._utime_from_header = self.config.getdefaultboolean( self.repoconfname, "utime_from_header", utime_from_header_global) # What do we substitute pathname separator in names (if any) self.sep_subst = '-' if os.path.sep == self.sep_subst: self.sep_subst = '_' # Interface from BaseFolder def getfullname(self): """Return the absolute file path to the Maildir folder (sans cur|new)""" return self._fullname # Interface from BaseFolder def get_uidvalidity(self): """Retrieve the current connections UIDVALIDITY value Maildirs have no notion of uidvalidity, so we just return a magic token.""" return 42 def _iswithintime(self, messagename, date): """Check to see if the given message is newer than date (a time_struct) according to the maildir name which should begin with a timestamp.""" timestampmatch = re_timestampmatch.search(messagename) if not timestampmatch: return True timestampstr = timestampmatch.group() timestamplong = int(timestampstr) if(timestamplong < time.mktime(date)): return False else: return True def _parse_filename(self, filename): """Returns a messages file name components Receives the file name (without path) of a msg. Usual format is '<%d_%d.%d.%s>,U=<%d>,FMD5=<%s>:2,' (pointy brackets denoting the various components). If FMD5 does not correspond with the current folder MD5, we will return None for the UID & FMD5 (as it is not valid in this folder). If UID or FMD5 can not be detected, we return `None` for the respective element. If flags are empty or cannot be detected, we return an empty flags list. :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" type. flags is a set() of Maildir flags. """ prefix, uid, fmd5, flags = None, None, None, set() prefixmatch = self.re_prefixmatch.match(filename) if prefixmatch: prefix = prefixmatch.group(1) folderstr = ',FMD5=%s'% self._foldermd5 foldermatch = folderstr in filename # If there was no folder MD5 specified, or if it mismatches, # assume it is a foreign (new) message and ret: uid, fmd5 = None, None # XXX: This is wrong behaviour: if FMD5 is missing or mismatches, assume # the mail is new and **fix UID to None** to avoid any conflict. # XXX: If UID is missing, I have no idea what FMD5 can do. Should be # fixed to None in this case, too. if foldermatch: uidmatch = re_uidmatch.search(filename) if uidmatch: uid = int(uidmatch.group(1)) flagmatch = self.re_flagmatch.search(filename) if flagmatch: flags = set((c for c in flagmatch.group(1))) return prefix, uid, fmd5, flags def _scanfolder(self, min_date=None, min_uid=None): """Cache the message list from a Maildir. If min_date is set, this finds the min UID of all messages newer than min_date and uses it as the real cutoff for considering messages. This handles the edge cases where the date is much earlier than messages with similar UID's (e.g. the UID was reassigned much later). Maildir flags are: D (draft) F (flagged) R (replied) S (seen) T (trashed), plus lower-case letters for custom flags. :returns: dict that can be used as self.messagelist. """ maxsize = self.getmaxsize() retval = {} files = [] nouidcounter = -1 # Messages without UIDs get negative UIDs. for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) files.extend((dirannex, filename) for filename in os.listdir(fulldirname)) date_excludees = {} for dirannex, filename in files: if filename.startswith('.'): continue # Ignore dot files. # We store just dirannex and filename, ie 'cur/123...' filepath = os.path.join(dirannex, filename) # Check maxsize if this message should be considered. if maxsize and (os.path.getsize( os.path.join(self.getfullname(), filepath)) > maxsize): continue prefix, uid, fmd5, flags = self._parse_filename(filename) if uid is None: # Assign negative uid to upload it. uid = nouidcounter nouidcounter -= 1 else: # It comes from our folder. uidmatch = re_uidmatch.search(filename) uid = None if not uidmatch: uid = nouidcounter nouidcounter -= 1 else: uid = int(uidmatch.group(1)) if min_uid != None and uid > 0 and uid < min_uid: continue if min_date != None and not self._iswithintime(filename, min_date): # Keep track of messages outside of the time limit, because they # still might have UID > min(UIDs of within-min_date). We hit # this case for maxage if any message had a known/valid datetime # and was re-uploaded because the UID in the filename got lost # (e.g. local copy/move). On next sync, it was assigned a new # UID from the server and will be included in the SEARCH # condition. So, we must re-include them later in this method # in order to avoid inconsistent lists of messages. date_excludees[uid] = self.msglist_item_initializer(uid) date_excludees[uid]['flags'] = flags date_excludees[uid]['filename'] = filepath else: # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S retval[uid] = self.msglist_item_initializer(uid) retval[uid]['flags'] = flags retval[uid]['filename'] = filepath if min_date != None: # Re-include messages with high enough uid's. positive_uids = [uid for uid in retval if uid > 0] if positive_uids: min_uid = min(positive_uids) for uid in date_excludees.keys(): if uid > min_uid: # This message was originally excluded because of # its date. It is re-included now because we want all # messages with UID > min_uid. retval[uid] = date_excludees[uid] return retval # Interface from BaseFolder def quickchanged(self, statusfolder): """Returns True if the Maildir has changed Assumes cachemessagelist() has already been called """ # Folder has different uids than statusfolder => TRUE. if sorted(self.getmessageuidlist()) != \ sorted(statusfolder.getmessageuidlist()): return True # Also check for flag changes, it's quick on a Maildir. for (uid, message) in self.getmessagelist().items(): if message['flags'] != statusfolder.getmessageflags(uid): return True return False # Nope, nothing changed. # Interface from BaseFolder def msglist_item_initializer(self, uid): return {'flags': set(), 'filename': '/no-dir/no-such-file/'} # Interface from BaseFolder def cachemessagelist(self, min_date=None, min_uid=None): if self.ismessagelistempty(): self.ui.loadmessagelist(self.repository, self) self.messagelist = self._scanfolder(min_date=min_date, min_uid=min_uid) self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) # Interface from BaseFolder def getmessage(self, uid): """Return the content of the message.""" filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) file = open(filepath, 'rt') retval = file.read() file.close() #TODO: WHY are we replacing \r\n with \n here? And why do we # read it as text? return retval.replace("\r\n", "\n") # Interface from BaseFolder def getmessagetime(self, uid): filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) def new_message_filename(self, uid, flags=set(), date=None): """Creates a new unique Maildir filename :param uid: The UID`None`, or a set of maildir flags :param flags: A set of maildir flags :returns: String containing unique message filename""" timeval, timeseq = _gettimeseq(date) uniq_name = '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ (timeval, timeseq, os.getpid(), socket.gethostname(), uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) return uniq_name.replace(os.path.sep, self.sep_subst) def save_to_tmp_file(self, filename, content): """Saves given content to the named temporary file in the 'tmp' subdirectory of $CWD. Arguments: - filename: name of the temporary file; - content: data to be saved. Returns: relative path to the temporary file that was created.""" tmpname = os.path.join('tmp', filename) # Open file and write it out. # XXX: why do we need to loop 7 times? tries = 7 while tries: tries = tries - 1 try: fd = os.open(os.path.join(self.getfullname(), tmpname), os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666) break except OSError as e: if not hasattr(e, 'EEXIST'): raise if e.errno == e.EEXIST: if tries: time.sleep(0.23) continue severity = OfflineImapError.ERROR.MESSAGE six.reraise(OfflineImapError, OfflineImapError( "Unique filename %s already exists."% filename, severity), exc_info()[2]) else: raise fd = os.fdopen(fd, 'wt') fd.write(content) # Make sure the data hits the disk. fd.flush() if self.dofsync(): os.fsync(fd) fd.close() return tmpname # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode.""" # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. self.ui.savemessage('maildir', uid, flags, self) if uid < 0: # We cannot assign a new uid. return uid if uid in self.messagelist: # We already have it, just update flags. self.savemessageflags(uid, flags) return uid # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') # Use the mail timestamp given by either Date or Delivery-date mail # headers. message_timestamp = None if self._filename_use_mail_timestamp is not False: try: message_timestamp = emailutil.get_message_date(content, 'Date') if message_timestamp is None: # Give a try with Delivery-date message_timestamp = emailutil.get_message_date( content, 'Delivery-date') except Exception as e: # This should never happen. from email.Parser import Parser from offlineimap.ui import getglobalui datestr = Parser().parsestr(content, True).get("Date") ui = getglobalui() ui.warn("UID %d has invalid date %s: %s\n" "Not using message timestamp as file prefix"% (uid, datestr, e)) # No need to check if message_timestamp is None here since it # would be overridden by _gettimeseq. messagename = self.new_message_filename(uid, flags, date=message_timestamp) tmpname = self.save_to_tmp_file(messagename, content) if self._utime_from_header is True: try: date = emailutil.get_message_date(content, 'Date') if date is not None: os.utime(os.path.join(self.getfullname(), tmpname), (date, date)) # In case date is wrongly so far into the future as to be > max # int32. except Exception as e: from email.Parser import Parser from offlineimap.ui import getglobalui datestr = Parser().parsestr(content, True).get("Date") ui = getglobalui() ui.warn("UID %d has invalid date %s: %s\n" "Not changing file modification time"% (uid, datestr, e)) self.messagelist[uid] = self.msglist_item_initializer(uid) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = tmpname # savemessageflags moves msg to 'cur' or 'new' as appropriate. self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid # Interface from BaseFolder def getmessageflags(self, uid): return self.messagelist[uid]['flags'] # Interface from BaseFolder def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. This function moves the message to the cur or new subdir, depending on the 'S'een flag. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" assert uid in self.messagelist oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) # If a message has been seen, it goes into 'cur' dir_prefix = 'cur' if 'S' in flags else 'new' if flags != self.messagelist[uid]['flags']: # Flags have actually changed, construct new filename Strip # off existing infostring infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off infostr = '%s2,%s'% (self.infosep, ''.join(sorted(flags))) filename += infostr newfilename = os.path.join(dir_prefix, filename) if (newfilename != oldfilename): try: os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) except OSError as e: six.reraise(OfflineImapError, OfflineImapError( "Can't rename file '%s' to '%s': %s"% (oldfilename, newfilename, e[1]), OfflineImapError.ERROR.FOLDER), exc_info()[2]) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename # Interface from BaseFolder def change_message_uid(self, uid, new_uid): """Change the message from existing uid to new_uid This will not update the statusfolder UID, you need to do that yourself. :param new_uid: (optional) If given, the old UID will be changed to a new UID. The Maildir backend can implement this as an efficient rename. """ if not uid in self.messagelist: raise OfflineImapError("Cannot change unknown Maildir UID %s"% uid, OfflineImapError.ERROR.MESSAGE) if uid == new_uid: return oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) flags = self.getmessageflags(uid) # TODO: we aren't keeping the prefix timestamp so we don't honor the # filename_use_mail_timestamp configuration option. newfilename = os.path.join(dir_prefix, self.new_message_filename(new_uid, flags)) os.rename(os.path.join(self.getfullname(), oldfilename), os.path.join(self.getfullname(), newfilename)) self.messagelist[new_uid] = self.messagelist[uid] self.messagelist[new_uid]['filename'] = newfilename del self.messagelist[uid] # Interface from BaseFolder def deletemessage(self, uid): """Unlinks a message file from the Maildir. :param uid: UID of a mail message :type uid: String :return: Nothing, or an Exception if UID but no corresponding file found. """ filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) try: os.unlink(filepath) except OSError: # Can't find the file -- maybe already deleted? newmsglist = self._scanfolder() if uid in newmsglist: # Nope, try new filename. filename = newmsglist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) def migratefmd5(self, dryrun=False): """Migrate FMD5 hashes from versions prior to 6.3.5 :param dryrun: Run in dry run mode :type fix: Boolean :return: None """ oldfmd5 = md5(self.name).hexdigest() msglist = self._scanfolder() for mkey, mvalue in msglist.items(): filename = os.path.join(self.getfullname(), mvalue['filename']) match = re.search("FMD5=([a-fA-F0-9]+)", filename) if match is None: self.ui.debug("maildir", "File `%s' doesn't have an FMD5 assigned" % filename) elif match.group(1) == oldfmd5: self.ui.info("Migrating file `%s' to FMD5 `%s'" % (filename, self._foldermd5)) if not dryrun: newfilename = filename.replace( "FMD5=" + match.group(1), "FMD5=" + self._foldermd5) try: os.rename(filename, newfilename) except OSError as e: six.reraise(OfflineImapError, OfflineImapError( "Can't rename file '%s' to '%s': %s"% (filename, newfilename, e[1]), OfflineImapError.ERROR.FOLDER), exc_info()[2]) elif match.group(1) != self._foldermd5: self.ui.warn(("Inconsistent FMD5 for file `%s':" " Neither `%s' nor `%s' found") % (filename, oldfmd5, self._foldermd5)) ================================================ FILE: offlineimap/folder/UIDMaps.py ================================================ # Base folder support # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os.path import shutil from os import fsync, unlink from sys import exc_info from threading import Lock import six from offlineimap import OfflineImapError from .IMAP import IMAPFolder class MappedIMAPFolder(IMAPFolder): """IMAP class to map between Folder() instances where both side assign a uid This Folder is used on the local side, while the remote side should be an IMAPFolder. Instance variables (self.): dryrun: boolean. r2l: dict mapping message uids: self.r2l[remoteuid]=localuid l2r: dict mapping message uids: self.r2l[localuid]=remoteuid #TODO: what is the difference, how are they used? diskr2l: dict mapping message uids: self.r2l[remoteuid]=localuid diskl2r: dict mapping message uids: self.r2l[localuid]=remoteuid""" def __init__(self, imapserver, name, repository, decode=True): IMAPFolder.__init__(self, imapserver, name, repository, decode=False) self.dryrun = self.config.getdefaultboolean("general", "dry-run", True) self.maplock = Lock() self.diskr2l, self.diskl2r = self._loadmaps() self.r2l, self.l2r = None, None # Representing the local IMAP Folder using local UIDs. # XXX: This should be removed since we inherit from IMAPFolder. # See commit 3ce514e92ba7 to know more. self._mb = IMAPFolder(imapserver, name, repository, decode=False) def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), self.getfolderbasename()) def _loadmaps(self): mapfilename = self._getmapfilename() mapfilenametmp = "%s.tmp"% mapfilename mapfilenamelock = "%s.lock"% mapfilename with self.maplock and open(mapfilenamelock, 'w') as mapfilelock: try: fnctl.lockf(mapfilelock, fnctl.LOCK_EX) # Blocks until acquired. except NameError: pass # Windows... if os.path.exists(mapfilenametmp): self.ui.warn("a previous run might have leave the UIDMaps file" " in incorrect state; some sync operations might be done" " again and some emails might become duplicated.") unlink(mapfilenametmp) if not os.path.exists(mapfilename): return ({}, {}) file = open(mapfilename, 'rt') r2l = {} l2r = {} while True: line = file.readline() if not len(line): break try: line = line.strip() except ValueError: six.reraise(Exception, Exception( "Corrupt line '%s' in UID mapping file '%s'"% (line, mapfilename)), exc_info()[2]) (str1, str2) = line.split(':') loc = int(str1) rem = int(str2) r2l[rem] = loc l2r[loc] = rem return (r2l, l2r) def _savemaps(self): if self.dryrun is True: return mapfilename = self._getmapfilename() # Do not use the map file directly to prevent from leaving it truncated. mapfilenametmp = "%s.tmp"% mapfilename mapfilenamelock = "%s.lock"% mapfilename with self.maplock and open(mapfilenamelock, 'w') as mapfilelock: # The "account" lock already prevents from multiple access by # different processes. However, we still need to protect for # multiple access from different threads. try: fnctl.lockf(mapfilelock, fnctl.LOCK_EX) # Blocks until acquired. except NameError: pass # Windows... with open(mapfilenametmp, 'wt') as mapfilefd: for (key, value) in self.diskl2r.items(): mapfilefd.write("%d:%d\n"% (key, value)) if self.dofsync(): fsync(mapfilefd) # The lock is released when the file descriptor ends. shutil.move(mapfilenametmp, mapfilename) def _uidlist(self, mapping, items): try: return [mapping[x] for x in items] except KeyError as e: six.reraise(OfflineImapError, OfflineImapError( "Could not find UID for msg '{0}' (f:'{1}'." " This is usually a bad thing and should be " "reported on the mailing list.".format( e.args[0], self), OfflineImapError.ERROR.MESSAGE), exc_info()[2]) # Interface from BaseFolder def cachemessagelist(self, min_date=None, min_uid=None): self._mb.cachemessagelist(min_date=min_date, min_uid=min_uid) reallist = self._mb.getmessagelist() self.messagelist = self._mb.messagelist with self.maplock: # OK. Now we've got a nice list. First, delete things from the # summary that have been deleted from the folder. for luid in self.diskl2r.keys(): if not luid in reallist: ruid = self.diskl2r[luid] #XXX: the following KeyError are sightly unexpected. This # would require more digging to understand how it's # possible. errorMessage = ("unexpected error: key {} was not found " "in memory, see " "https://github.com/OfflineIMAP/offlineimap/issues/445" " to know more." ) try: del self.diskr2l[ruid] except KeyError as e: self.ui.warn(errorMessage.format(ruid)) try: del self.diskl2r[luid] except KeyError as e: self.ui.warn(errorMessage.format(ruid)) # Now, assign negative UIDs to local items. self._savemaps() nextneg = -1 self.r2l = self.diskr2l.copy() self.l2r = self.diskl2r.copy() for luid in reallist.keys(): if not luid in self.l2r: ruid = nextneg nextneg -= 1 self.l2r[luid] = ruid self.r2l[ruid] = luid def dropmessagelistcache(self): self._mb.dropmessagelistcache() # Interface from BaseFolder def uidexists(self, ruid): """Checks if the (remote) UID exists in this Folder""" # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return ruid in self.r2l # Interface from BaseFolder def getmessageuidlist(self): """Gets a list of (remote) UIDs. You may have to call cachemessagelist() before calling this function!""" # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return self.r2l.keys() # Interface from BaseFolder def getmessagecount(self): """Gets the number of messages in this folder. You may have to call cachemessagelist() before calling this function!""" # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return len(self.r2l) # Interface from BaseFolder def getmessagelist(self): """Gets the current message list. This function's implementation is quite expensive for the mapped UID case. You must call cachemessagelist() before calling this function!""" retval = {} localhash = self._mb.getmessagelist() with self.maplock: for key, value in list(localhash.items()): try: key = self.l2r[key] except KeyError: # Sometimes, the IMAP backend may put in a new message, # then this function acquires the lock before the system # has the chance to note it in the mapping. In that case, # just ignore it. continue value = value.copy() value['uid'] = self.l2r[value['uid']] retval[key] = value return retval # Interface from BaseFolder def getmessage(self, uid): """Returns the content of the specified message.""" return self._mb.getmessage(self.r2l[uid]) # Interface from BaseFolder def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. The UIDMaps class will not return a newly assigned uid, as it internally maps different uids between IMAP servers. So a successful savemessage() invocation will return the same uid it has been invoked with. As it maps between 2 IMAP servers which means the source message must already have an uid, it requires a positive uid to be passed in. Passing in a message with a negative uid will do nothing and return the negative uid. If the uid is > 0, the backend should set the uid to this, if it can. If it cannot set the uid to that, it will save it anyway. It will return the uid assigned in any case. See folder/Base for details. Note that savemessage() does not check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode. """ self.ui.savemessage('imap', uid, flags, self) # Mapped UID instances require the source to already have a # positive UID, so simply return here. if uid < 0: return uid # If msg uid already exists, just modify the flags. if uid in self.r2l: self.savemessageflags(uid, flags) return uid newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: raise OfflineImapError("server of repository '%s' did not return " "a valid UID (got '%s') for UID '%s' from '%s'"% ( self._mb.getname(), newluid, uid, self.getname() ), OfflineImapError.ERROR.MESSAGE ) with self.maplock: self.diskl2r[newluid] = uid self.diskr2l[uid] = newluid self.l2r[newluid] = uid self.r2l[uid] = newluid self._savemaps() return uid # Interface from BaseFolder def getmessageflags(self, uid): return self._mb.getmessageflags(self.r2l[uid]) # Interface from BaseFolder def getmessagetime(self, uid): return None # Interface from BaseFolder def savemessageflags(self, uid, flags): """Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" self._mb.savemessageflags(self.r2l[uid], flags) # Interface from BaseFolder def addmessageflags(self, uid, flags): self._mb.addmessageflags(self.r2l[uid], flags) # Interface from BaseFolder def addmessagesflags(self, uidlist, flags): self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) # Interface from BaseFolder def change_message_uid(self, ruid, new_ruid): """Change the message from existing ruid to new_ruid :param new_uid: The old remote UID will be changed to a new UID. The UIDMaps case handles this efficiently by simply changing the mappings file.""" if ruid not in self.r2l: raise OfflineImapError("Cannot change unknown Maildir UID %s"% ruid, OfflineImapError.ERROR.MESSAGE) if ruid == new_ruid: return # sanity check shortcut with self.maplock: luid = self.r2l[ruid] self.l2r[luid] = new_ruid del self.r2l[ruid] self.r2l[new_ruid] = luid # TODO: diskl2r|r2l are a pain to sync and should be done away with # diskl2r only contains positive UIDs, so wrap in ifs. if luid > 0: self.diskl2r[luid] = new_ruid if ruid > 0: del self.diskr2l[ruid] if new_ruid > 0: self.diskr2l[new_ruid] = luid self._savemaps() def _mapped_delete(self, uidlist): with self.maplock: needssave = 0 for ruid in uidlist: luid = self.r2l[ruid] del self.r2l[ruid] del self.l2r[luid] if ruid > 0: del self.diskr2l[ruid] del self.diskl2r[luid] needssave = 1 if needssave: self._savemaps() # Interface from BaseFolder def deletemessageflags(self, uid, flags): self._mb.deletemessageflags(self.r2l[uid], flags) # Interface from BaseFolder def deletemessagesflags(self, uidlist, flags): self._mb.deletemessagesflags(self._uidlist(self.r2l, uidlist), flags) # Interface from BaseFolder def deletemessage(self, uid): self._mb.deletemessage(self.r2l[uid]) self._mapped_delete([uid]) # Interface from BaseFolder def deletemessages(self, uidlist): self._mb.deletemessages(self._uidlist(self.r2l, uidlist)) self._mapped_delete(uidlist) ================================================ FILE: offlineimap/folder/__init__.py ================================================ from . import Base, Gmail, IMAP, Maildir, LocalStatus, UIDMaps ================================================ FILE: offlineimap/globals.py ================================================ # Copyright 2013-2016 Eygene A. Ryabinkin & contributors. # # Module that holds various global objects. from offlineimap.utils import const # Holds command-line options for OfflineIMAP. options = const.ConstProxy() def set_options(source): """Sets the source for options variable.""" options.set_source(source) ================================================ FILE: offlineimap/imaplibutil.py ================================================ # imaplib utilities # Copyright (C) 2002-2016 John Goerzen & contributors # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import fcntl import time import subprocess import threading import rfc6555 import socket import errno import zlib from sys import exc_info from hashlib import sha512, sha384, sha256, sha224, sha1 import six from offlineimap import OfflineImapError from offlineimap.ui import getglobalui from offlineimap.virtual_imaplib2 import IMAP4, IMAP4_SSL, InternalDate, Mon2num class UsefulIMAPMixIn(object): def __getselectedfolder(self): if self.state == 'SELECTED': return self.mailbox return None def select(self, mailbox='INBOX', readonly=False, force=False): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already selected or raises an :exc:`OfflineImapError`.""" if self.__getselectedfolder() == mailbox and \ self.is_readonly == readonly and \ not force: # No change; return. return try: result = super(UsefulIMAPMixIn, self).select(mailbox, readonly) except self.readonly as e: # pass self.readonly to our callers raise except self.abort as e: # self.abort is raised when we are supposed to retry errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\ "ver said: %s" % (self.host, mailbox, e.args[0]) severity = OfflineImapError.ERROR.FOLDER_RETRY six.reraise(OfflineImapError, OfflineImapError(errstr, severity), exc_info()[2]) if result[0] != 'OK': #in case of error, bail out with OfflineImapError errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\ (mailbox, result) severity = OfflineImapError.ERROR.FOLDER raise OfflineImapError(errstr, severity) return result # Overrides private function from IMAP4 (@imaplib2) def _mesg(self, s, tn=None, secs=None): new_mesg(self, s, tn, secs) # Overrides private function from IMAP4 (@imaplib2) def open_socket(self): """open_socket() Open socket choosing first address family available.""" if self.af == socket.AF_UNSPEC: # happy-eyeballs! return rfc6555.create_connection((self.host, self.port)) else: return self._open_socket_for_af(self.af) def _open_socket_for_af(self, af): msg = (-1, 'could not open socket') for res in socket.getaddrinfo(self.host, self.port, af, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: # use socket of our own, possiblly socksified socket. s = self.socket(af, socktype, proto) except socket.error as msg: continue try: for i in (0, 1): try: s.connect(sa) break except socket.error as msg: if len(msg.args) < 2 or msg.args[0] != errno.EINTR: raise else: raise socket.error(msg) except socket.error as msg: s.close() continue break else: raise socket.error(msg) return s class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4): """IMAP4 client class over a tunnel Instantiate with: IMAP4_Tunnel(tunnelcmd) tunnelcmd -- shell command to generate the tunnel. The result will be in PREAUTH stage.""" def __init__(self, tunnelcmd, **kwargs): if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] IMAP4.__init__(self, tunnelcmd, **kwargs) def open(self, host, port): """The tunnelcmd comes in on host!""" self.host = host self.process = subprocess.Popen(host, shell=True, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (self.outfd, self.infd) = (self.process.stdin, self.process.stdout) # imaplib2 polls on this fd self.read_fd = self.infd.fileno() self.set_nonblocking(self.read_fd) def set_nonblocking(self, fd): """Mark fd as nonblocking""" # get the file's current flag settings fl = fcntl.fcntl(fd, fcntl.F_GETFL) # clear non-blocking mode from flags fl = fl & ~os.O_NONBLOCK fcntl.fcntl(fd, fcntl.F_SETFL, fl) def read(self, size): """data = read(size) Read at most 'size' bytes from remote.""" if self.decompressor is None: return os.read(self.read_fd, size) if self.decompressor.unconsumed_tail: data = self.decompressor.unconsumed_tail else: data = os.read(self.read_fd, 8192) return self.decompressor.decompress(data, size) def send(self, data): if self.compressor is not None: data = self.compressor.compress(data) data += self.compressor.flush(zlib.Z_SYNC_FLUSH) self.outfd.write(data) def shutdown(self): self.infd.close() self.outfd.close() self.process.wait() def new_mesg(self, s, tn=None, secs=None): if secs is None: secs = time.time() if tn is None: tn = threading.currentThread().getName() tm = time.strftime('%M:%S', time.localtime(secs)) getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s)) class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL): """Improved version of imaplib.IMAP4_SSL overriding select().""" def __init__(self, *args, **kwargs): if "af" in kwargs: self.af = kwargs['af'] del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] self._fingerprint = kwargs.get('fingerprint', None) if type(self._fingerprint) != type([]): self._fingerprint = [self._fingerprint] if 'fingerprint' in kwargs: del kwargs['fingerprint'] super(WrappedIMAP4_SSL, self).__init__(*args, **kwargs) def open(self, host=None, port=None): if not self.ca_certs and not self._fingerprint: raise OfflineImapError("No CA certificates " "and no server fingerprints configured. " "You must configure at least something, otherwise " "having SSL helps nothing.", OfflineImapError.ERROR.REPO) super(WrappedIMAP4_SSL, self).open(host, port) if self._fingerprint: server_cert = self.sock.getpeercert(True) hashes = sha512, sha384, sha256, sha224, sha1 server_fingerprints = [hash(server_cert).hexdigest() for hash in hashes] # compare fingerprints matches = [(server_fingerprint in self._fingerprint) for server_fingerprint in server_fingerprints] if not any(matches): raise OfflineImapError("Server SSL fingerprint(s) '%s' " "for hostname '%s' " "does not match configured fingerprint(s) %s. " "Please verify and set 'cert_fingerprint' accordingly " "if not set yet."% (zip([hash.__name__ for hash in hashes], server_fingerprints), host, self._fingerprint), OfflineImapError.ERROR.REPO) class WrappedIMAP4(UsefulIMAPMixIn, IMAP4): """Improved version of imaplib.IMAP4 overriding select().""" def __init__(self, *args, **kwargs): if "af" in kwargs: self.af = kwargs['af'] del kwargs['af'] if "use_socket" in kwargs: self.socket = kwargs['use_socket'] del kwargs['use_socket'] IMAP4.__init__(self, *args, **kwargs) def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. Returns seconds since the epoch.""" from calendar import timegm mo = InternalDate.match(resp) if not mo: return None mon = Mon2num[mo.group('mon')] zonen = mo.group('zonen') day = int(mo.group('day')) year = int(mo.group('year')) hour = int(mo.group('hour')) min = int(mo.group('min')) sec = int(mo.group('sec')) zoneh = int(mo.group('zoneh')) zonem = int(mo.group('zonem')) # INTERNALDATE timezone must be subtracted to get UT zone = (zoneh*60 + zonem)*60 if zonen == '-': zone = -zone tt = (year, mon, day, hour, min, sec, -1, -1, -1) return timegm(tt) - zone ================================================ FILE: offlineimap/imapserver.py ================================================ # IMAP server support # Copyright (C) 2002-2018 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import datetime import hmac import socket import json import urllib import time import errno import socket from socket import gaierror from sys import exc_info from ssl import SSLError, cert_time_to_seconds from threading import Lock, BoundedSemaphore, Thread, Event, currentThread import six import offlineimap.accounts from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError from offlineimap.ui import getglobalui try: import gssapi have_gss = True except ImportError: have_gss = False class IMAPServer(object): """Initializes all variables from an IMAPRepository() instance Various functions, such as acquireconnection() return an IMAP4 object on which we can operate. Public instance variables are: self.: delim The server's folder delimiter. Only valid after acquireconnection() """ def __init__(self, repos): """:repos: a IMAPRepository instance.""" self.ui = getglobalui() self.repos = repos self.config = repos.getconfig() self.preauth_tunnel = repos.getpreauthtunnel() self.transport_tunnel = repos.gettransporttunnel() if self.preauth_tunnel and self.transport_tunnel: raise OfflineImapError('%s: '% repos + 'you must enable precisely one ' 'type of tunnel (preauth or transport), ' 'not both', OfflineImapError.ERROR.REPO) self.tunnel = \ self.preauth_tunnel if self.preauth_tunnel \ else self.transport_tunnel self.username = \ None if self.preauth_tunnel else repos.getuser() self.user_identity = repos.get_remote_identity() self.authmechs = repos.get_auth_mechanisms() self.password = None self.passworderror = None self.goodpassword = None self.usessl = repos.getssl() self.useipv6 = repos.getipv6() if self.useipv6 is True: self.af = socket.AF_INET6 elif self.useipv6 is False: self.af = socket.AF_INET else: self.af = socket.AF_UNSPEC self.hostname = None if self.transport_tunnel or self.preauth_tunnel else repos.gethost() self.port = repos.getport() if self.port is None: self.port = 993 if self.usessl else 143 self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() if self.sslcacertfile is None: self.__verifycert = None # Disable cert verification. # This way of working sucks hard... self.fingerprint = repos.get_ssl_fingerprint() self.tlslevel = repos.gettlslevel() self.sslversion = repos.getsslversion() self.starttls = repos.getstarttls() if self.usessl \ and self.tlslevel is not "tls_compat" \ and self.sslversion is None: raise Exception("When 'tls_level' is not 'tls_compat' " "the 'ssl_version' must be set explicitly.") self.oauth2_refresh_token = repos.getoauth2_refresh_token() self.oauth2_access_token = repos.getoauth2_access_token() self.oauth2_client_id = repos.getoauth2_client_id() self.oauth2_client_secret = repos.getoauth2_client_secret() self.oauth2_request_url = repos.getoauth2_request_url() self.oauth2_access_token_expires_at = None self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() self.reference = repos.getreference() self.idlefolders = repos.getidlefolders() self.gss_vc = None self.gssapi = False # In order to support proxy connection, we have to override the # default socket instance with our own socksified socket instance. # We add this option to bypass the GFW in China. self.proxied_socket = self._get_proxy('proxy', socket.socket) # Turns out that the GFW in China is no longer blocking imap.gmail.com # However accounts.google.com (for oauth2) definitey is. Therefore # it is not strictly necessary to use a proxy for *both* IMAP *and* # oauth2, so a new option is added: authproxy. # Set proxy for use in authentication (only) if desired. # If not set, is same as proxy option (compatible with current configs) # To use a proxied_socket but not an authproxied_socket # set authproxy = '' in config self.authproxied_socket = self._get_proxy('authproxy', self.proxied_socket) def _get_proxy(self, proxysection, dfltsocket): _account_section = 'Account ' + self.repos.account.name if not self.config.has_option(_account_section, proxysection): return dfltsocket proxy = self.config.get(_account_section, proxysection) if proxy == '': # explicitly set no proxy (overrides default return of dfltsocket) return socket.socket # Powered by PySocks. try: import socks proxy_type, host, port = proxy.split(":") port = int(port) socks.setdefaultproxy(getattr(socks, proxy_type), host, port) return socks.socksocket except ImportError: self.ui.warn("PySocks not installed, ignoring proxy option.") except (AttributeError, ValueError) as e: self.ui.warn("Bad proxy option %s for account %s: %s " "Ignoring %s option."% (proxy, self.repos.account.name, e, proxysection)) return dfltsocket def __getpassword(self): """Returns the server password or None""" if self.goodpassword != None: # use cached good one first return self.goodpassword if self.password != None and self.passworderror == None: return self.password # non-failed preconfigured one # get 1) configured password first 2) fall back to asking via UI self.password = self.repos.getpassword() or \ self.ui.getpass(self.username, self.config, self.passworderror) self.passworderror = None return self.password def __md5handler(self, response): challenge = response.strip() self.ui.debug('imap', '__md5handler: got challenge %s'% challenge) passwd = self.__getpassword() retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest() self.ui.debug('imap', '__md5handler: returning %s'% retval) return retval def __loginauth(self, imapobj): """ Basic authentication via LOGIN command.""" self.ui.debug('imap', 'Attempting IMAP LOGIN authentication') imapobj.login(self.username, self.__getpassword()) def __plainhandler(self, response): """Implements SASL PLAIN authentication, RFC 4616, http://tools.ietf.org/html/rfc4616""" authc = self.username if not authc: raise OfflineImapError("No username provided for '%s'" % self.repos.getname(), OfflineImapError.ERROR.REPO) passwd = self.__getpassword() authz = b'' if self.user_identity != None: authz = self.user_identity # At this point all authz, authc and passwd are expected bytes encoded # in UTF-8. NULL = b'\x00' retval = NULL.join((authz, authc, passwd)) logsafe_retval = NULL.join((authz, authc, b'(passwd hidden for log)')) self.ui.debug('imap', '__plainhandler: returning %s'% logsafe_retval) return retval def __xoauth2handler(self, response): now = datetime.datetime.now() if self.oauth2_access_token_expires_at \ and self.oauth2_access_token_expires_at < now: self.oauth2_access_token = None self.ui.debug('imap', 'xoauth2handler: oauth2_access_token expired') if self.oauth2_access_token is None: if self.oauth2_request_url is None: raise OfflineImapError("No remote oauth2_request_url for " "repository '%s' specified."% self, OfflineImapError.ERROR.REPO) # Generate new access token. params = {} params['client_id'] = self.oauth2_client_id params['client_secret'] = self.oauth2_client_secret params['refresh_token'] = self.oauth2_refresh_token params['grant_type'] = 'refresh_token' self.ui.debug('imap', 'xoauth2handler: url "%s"'% self.oauth2_request_url) self.ui.debug('imap', 'xoauth2handler: params "%s"'% params) original_socket = socket.socket socket.socket = self.authproxied_socket try: response = urllib.urlopen( self.oauth2_request_url, urllib.urlencode(params)).read() except Exception as e: try: msg = "%s (configuration is: %s)"% (e, str(params)) except Exception as eparams: msg = "%s [cannot display configuration: %s]"% (e, eparams) six.reraise(type(e), type(e)(msg), exc_info()[2]) finally: socket.socket = original_socket resp = json.loads(response) self.ui.debug('imap', 'xoauth2handler: response "%s"'% resp) if u'error' in resp: raise OfflineImapError("xoauth2handler got: %s"% resp, OfflineImapError.ERROR.REPO) self.oauth2_access_token = resp['access_token'] if u'expires_in' in resp: self.oauth2_access_token_expires_at = now + datetime.timedelta( seconds=resp['expires_in']/2 ) self.ui.debug('imap', 'xoauth2handler: access_token "%s expires %s"'% ( self.oauth2_access_token, self.oauth2_access_token_expires_at)) auth_string = 'user=%s\1auth=Bearer %s\1\1'% ( self.username, self.oauth2_access_token) #auth_string = base64.b64encode(auth_string) self.ui.debug('imap', 'xoauth2handler: returning "%s"'% auth_string) return auth_string # Perform the next step handling a GSSAPI connection. # Client sends first, so token will be ignored if there is no context. def __gsshandler(self, token): if token == "": token = None try: if not self.gss_vc: name = gssapi.Name('imap@' + self.hostname, gssapi.NameType.hostbased_service) self.gss_vc = gssapi.SecurityContext(usage="initiate", name=name) if not self.gss_vc.complete: response = self.gss_vc.step(token) return response if response else "" elif token is None: # uh... context is complete, so there's no negotiation we can # do. But we also don't have a token, so we can't send any # kind of response. Empirically, some (but not all) servers # seem to put us in this state, and seem fine with getting no # GSSAPI content in response, so give it to them. return "" # Don't bother checking qop because we're over a TLS channel # already. But hey, if some server started encrypting tomorrow, # we'd be ready since krb5 always requests integrity and # confidentiality support. response = self.gss_vc.unwrap(token) # This is a behavior we got from pykerberos. First byte is one, # first four bytes are preserved (pykerberos calls this a length). # Any additional bytes are username. reply = [] reply[0:4] = response.message[0:4] reply[0] = '\x01' if self.username: reply[5:] = self.username reply = ''.join(reply) response = self.gss_vc.wrap(reply, response.encrypted) return response.message if response.message else "" except gssapi.exceptions.GSSError as err: # GSSAPI errored out on us; respond with None to cancel the # authentication self.ui.debug('imap', err.gen_message()) return None def __start_tls(self, imapobj): if 'STARTTLS' in imapobj.capabilities and not self.usessl: self.ui.debug('imap', 'Using STARTTLS connection') try: imapobj.starttls() except imapobj.error as e: raise OfflineImapError("Failed to start " "TLS connection: %s"% str(e), OfflineImapError.ERROR.REPO, None, exc_info()[2]) ## All __authn_* procedures are helpers that do authentication. ## They are class methods that take one parameter, IMAP object. ## ## Each function should return True if authentication was ## successful and False if authentication wasn't even tried ## for some reason (but not when IMAP has no such authentication ## capability, calling code checks that). ## ## Functions can also raise exceptions; two types are special ## and will be handled by the calling code: ## ## - imapobj.error means that there was some error that ## comes from imaplib2; ## ## - OfflineImapError means that function detected some ## problem by itself. def __authn_gssapi(self, imapobj): if not have_gss: return False self.connectionlock.acquire() try: imapobj.authenticate('GSSAPI', self.__gsshandler) return True except imapobj.error as e: self.gssapi = False raise else: self.gssapi = True self.gss_vc = None finally: self.connectionlock.release() def __authn_cram_md5(self, imapobj): imapobj.authenticate('CRAM-MD5', self.__md5handler) return True def __authn_plain(self, imapobj): imapobj.authenticate('PLAIN', self.__plainhandler) return True def __authn_xoauth2(self, imapobj): if self.oauth2_refresh_token is None \ and self.oauth2_access_token is None: return False imapobj.authenticate('XOAUTH2', self.__xoauth2handler) return True def __authn_login(self, imapobj): # Use LOGIN command, unless LOGINDISABLED is advertized # (per RFC 2595) if 'LOGINDISABLED' in imapobj.capabilities: raise OfflineImapError("IMAP LOGIN is " "disabled by server. Need to use SSL?", OfflineImapError.ERROR.REPO) else: self.__loginauth(imapobj) return True def __authn_helper(self, imapobj): """Authentication machinery for self.acquireconnection(). Raises OfflineImapError() of type ERROR.REPO when there are either fatal problems or no authentications succeeded. If any authentication method succeeds, routine should exit: warnings for failed methods are to be produced in the respective except blocks.""" # Stack stores pairs of (method name, exception) exc_stack = [] tried_to_authn = False tried_tls = False # Authentication routines, hash keyed by method name # with value that is a tuple with # - authentication function, # - tryTLS flag, # - check IMAP capability flag. auth_methods = { "GSSAPI": (self.__authn_gssapi, False, True), "XOAUTH2": (self.__authn_xoauth2, True, True), "CRAM-MD5": (self.__authn_cram_md5, True, True), "PLAIN": (self.__authn_plain, True, True), "LOGIN": (self.__authn_login, True, False), } # GSSAPI is tried first by default: we will probably go TLS after it and # GSSAPI mustn't be tunneled over TLS. for m in self.authmechs: if m not in auth_methods: raise Exception("Bad authentication method %s, " "please, file OfflineIMAP bug" % m) func, tryTLS, check_cap = auth_methods[m] # TLS must be initiated before checking capabilities: # they could have been changed after STARTTLS. if tryTLS and self.starttls and not tried_tls: tried_tls = True self.__start_tls(imapobj) if check_cap: cap = "AUTH=" + m if cap not in imapobj.capabilities: continue tried_to_authn = True self.ui.debug('imap', u'Attempting ' '%s authentication'% m) try: if func(imapobj): return except (imapobj.error, OfflineImapError) as e: self.ui.warn('%s authentication failed: %s'% (m, e)) exc_stack.append((m, e)) if len(exc_stack): msg = "\n\t".join([": ".join((x[0], str(x[1]))) for x in exc_stack]) raise OfflineImapError("All authentication types " "failed:\n\t%s"% msg, OfflineImapError.ERROR.REPO) if not tried_to_authn: methods = ", ".join([x[5:] for x in [x for x in imapobj.capabilities if x[0:5] == "AUTH="]]) raise OfflineImapError(u"Repository %s: no supported " "authentication mechanisms found; configured %s, " "server advertises %s"% (self.repos, ", ".join(self.authmechs), methods), OfflineImapError.ERROR.REPO) def __verifycert(self, cert, hostname): """Verify that cert (in socket.getpeercert() format) matches hostname. CRLs are not handled. Returns error message if any problems are found and None on success.""" errstr = "CA Cert verifying failed: " if not cert: return ('%s no certificate received'% errstr) dnsname = hostname.lower() certnames = [] # cert expired? notafter = cert.get('notAfter') if notafter: if time.time() >= cert_time_to_seconds(notafter): return '%s certificate expired %s'% (errstr, notafter) # First read commonName for s in cert.get('subject', []): key, value = s[0] if key == 'commonName': certnames.append(value.lower()) if len(certnames) == 0: return ('%s no commonName found in certificate'% errstr) # Then read subjectAltName for key, value in cert.get('subjectAltName', []): if key == 'DNS': certnames.append(value.lower()) # And finally try to match hostname with one of these names for certname in certnames: if (certname == dnsname or '.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]): return None return ('%s no matching domain name found in certificate'% errstr) def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data # Must be careful here that if we fail we should bail out gracefully # and release locks / threads so that the next attempt can try... success = False try: while success is not True: # Generate a new connection. if self.tunnel: self.ui.connecting( self.repos.getname(), 'tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel( self.tunnel, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, ) success = True elif self.usessl: self.ui.connecting( self.repos.getname(), self.hostname, self.port) self.ui.debug('imap', "%s: level '%s', version '%s'"% (self.repos.getname(), self.tlslevel, self.sslversion)) imapobj = imaplibutil.WrappedIMAP4_SSL( host=self.hostname, port=self.port, keyfile=self.sslclientkey, certfile=self.sslclientcert, ca_certs=self.sslcacertfile, cert_verify_cb=self.__verifycert, ssl_version=self.sslversion, timeout=socket.getdefaulttimeout(), fingerprint=self.fingerprint, use_socket=self.proxied_socket, tls_level=self.tlslevel, af=self.af, ) else: self.ui.connecting( self.repos.getname(), self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4( self.hostname, self.port, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, af=self.af, ) if not self.preauth_tunnel: try: self.__authn_helper(imapobj) self.goodpassword = self.password success = True except OfflineImapError as e: self.passworderror = str(e) raise # Enable compression if self.repos.getconfboolean('usecompression', 0): imapobj.enable_compression() # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'"% \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) with self.connectionlock: self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, SSLError) and e.errno == errno.EPERM: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port. Got: %s"% ( self.hostname, self.port, e) else: reason = "Unknown SSL protocol connecting to host '%s' for "\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network."% (self.hostname, self.port, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": six.reraise(OfflineImapError, OfflineImapError( "Could not connect to remote server '%s' " "for repository '%s'. Remote does not answer."% (self.hostname, self.repos), OfflineImapError.ERROR.REPO), exc_info()[2]) else: # re-raise all other errors raise def connectionwait(self): """Waits until there is a connection available. Note that between the time that a connection becomes available and the time it is requested, another thread may have grabbed it. This function is mainly present as a way to avoid spawning thousands of threads to copy messages, then have them all wait for 3 available connections. It's OK if we have maxconnections + 1 or 2 threads, which is what this will help us do.""" self.semaphore.acquire() # Blocking until maxconnections has free slots. self.semaphore.release() def close(self): # Make sure I own all the semaphores. Let the threads finish # their stuff. This is a blocking method. with self.connectionlock: # first, wait till all connections had been released. # TODO: won't work IMHO, as releaseconnection() also # requires the connectionlock, leading to a potential # deadlock! Audit & check! threadutil.semaphorereset(self.semaphore, self.maxconnections) for imapobj in self.assignedconnections + self.availableconnections: imapobj.logout() self.assignedconnections = [] self.availableconnections = [] self.lastowner = {} # reset GSSAPI state self.gss_vc = None self.gssapi = False def keepalive(self, timeout, event): """Sends a NOOP to each connection recorded. It will wait a maximum of timeout seconds between doing this, and will continue to do so until the Event object as passed is true. This method is expected to be invoked in a separate thread, which should be join()'d after the event is set.""" self.ui.debug('imap', 'keepalive thread started') while not event.isSet(): self.connectionlock.acquire() numconnections = len(self.assignedconnections) + \ len(self.availableconnections) self.connectionlock.release() threads = [] for i in range(numconnections): self.ui.debug('imap', 'keepalive: processing connection %d of %d'% (i, numconnections)) if len(self.idlefolders) > i: # IDLE thread idler = IdleThread(self, self.idlefolders[i]) else: # NOOP thread idler = IdleThread(self) idler.start() threads.append(idler) self.ui.debug('imap', 'keepalive: waiting for timeout') event.wait(timeout) self.ui.debug('imap', 'keepalive: after wait') for idler in threads: # Make sure all the commands have completed. idler.stop() idler.join() self.ui.debug('imap', 'keepalive: all threads joined') self.ui.debug('imap', 'keepalive: event is set; exiting') return def releaseconnection(self, connection, drop_conn=False): """Releases a connection, returning it to the pool. :param drop_conn: If True, the connection will be released and not be reused. This can be used to indicate broken connections.""" if connection is None: return # Noop on bad connection. self.connectionlock.acquire() self.assignedconnections.remove(connection) # Don't reuse broken connections if connection.Terminate or drop_conn: connection.logout() else: self.availableconnections.append(connection) self.connectionlock.release() self.semaphore.release() class IdleThread(object): def __init__(self, parent, folder=None): """If invoked without 'folder', perform a NOOP and wait for self.stop() to be called. If invoked with folder, switch to IDLE mode and synchronize once we have a new message""" self.parent = parent self.folder = folder self.stop_sig = Event() self.ui = getglobalui() if folder is None: self.thread = Thread(target=self.noop) else: self.thread = Thread(target=self.__idle) self.thread.setDaemon(1) def start(self): self.thread.start() def stop(self): self.stop_sig.set() def join(self): self.thread.join() def noop(self): # TODO: AFAIK this is not optimal, we will send a NOOP on one # random connection (ie not enough to keep all connections # open). In case we do the noop multiple times, we can well use # the same connection every time, as we get a random one. This # function should IMHO send a noop on ALL available connections # to the server. imapobj = self.parent.acquireconnection() try: imapobj.noop() except imapobj.abort: self.ui.warn('Attempting NOOP on dropped connection %s'% imapobj.identifier) self.parent.releaseconnection(imapobj, True) imapobj = None finally: if imapobj: self.parent.releaseconnection(imapobj) self.stop_sig.wait() # wait until we are supposed to quit def __dosync(self): remoterepos = self.parent.repos account = remoterepos.account localrepos = account.localrepos remoterepos = account.remoterepos statusrepos = account.statusrepos remotefolder = remoterepos.getfolder(self.folder, decode=False) hook = account.getconf('presynchook', '') account.callhook(hook) offlineimap.accounts.syncfolder(account, remotefolder, quick=False) hook = account.getconf('postsynchook', '') account.callhook(hook) ui = getglobalui() ui.unregisterthread(currentThread()) #syncfolder registered the thread def __idle(self): """Invoke IDLE mode until timeout or self.stop() is invoked.""" def callback(args): """IDLE callback function invoked by imaplib2. This is invoked when a) The IMAP server tells us something while in IDLE mode, b) we get an Exception (e.g. on dropped connections, or c) the standard imaplib IDLE timeout of 29 minutes kicks in.""" result, cb_arg, exc_data = args if exc_data is None and not self.stop_sig.isSet(): # No Exception, and we are not supposed to stop: self.needsync = True self.stop_sig.set() # Continue to sync. def noop(imapobj): """Factorize the noop code.""" try: # End IDLE mode with noop, imapobj can point to a dropped conn. imapobj.noop() except imapobj.abort: self.ui.warn('Attempting NOOP on dropped connection %s'% imapobj.identifier) self.parent.releaseconnection(imapobj, True) else: self.parent.releaseconnection(imapobj) while not self.stop_sig.isSet(): self.needsync = False success = False # Successfully selected FOLDER? while not success: imapobj = self.parent.acquireconnection() try: imapobj.select(self.folder) except OfflineImapError as e: if e.severity == OfflineImapError.ERROR.FOLDER_RETRY: # Connection closed, release connection and retry. self.ui.error(e, exc_info()[2]) self.parent.releaseconnection(imapobj, True) elif e.severity == OfflineImapError.ERROR.FOLDER: # Just continue the process on such error for now. self.ui.error(e, exc_info()[2]) else: # Stops future attempts to sync this account. raise else: success = True if "IDLE" in imapobj.capabilities: imapobj.idle(callback=callback) else: self.ui.warn("IMAP IDLE not supported on server '%s'." "Sleep until next refresh cycle."% imapobj.identifier) noop(imapobj) #XXX: why? self.stop_sig.wait() # self.stop() or IDLE callback are invoked. noop(imapobj) if self.needsync: # Here not via self.stop, but because IDLE responded. Do # another round and invoke actual syncing. self.stop_sig.clear() self.__dosync() ================================================ FILE: offlineimap/imaputil.py ================================================ # IMAP utility module # Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import re import string import binascii import codecs from offlineimap.ui import getglobalui ## Globals # Message headers that use space as the separator (for label storage) SPACE_SEPARATED_LABEL_HEADERS = ('X-Label', 'Keywords') # Find the modified UTF-7 shifts of an international mailbox name. MUTF7_SHIFT_RE = re.compile(r'&[^-]*-|\+') def __debug(*args): msg = [] for arg in args: msg.append(str(arg)) getglobalui().debug('imap', " ".join(msg)) def dequote(s): """Takes string which may or may not be quoted and unquotes it. It only considers double quotes. This function does NOT consider parenthised lists to be quoted.""" if s and s.startswith('"') and s.endswith('"'): s = s[1:-1] # Strip off the surrounding quotes. s = s.replace('\\"', '"') s = s.replace('\\\\', '\\') return s def quote(s): """Takes an unquoted string and quotes it. It only adds double quotes. This function does NOT consider parenthised lists to be quoted.""" s = s.replace('"', '\\"') s = s.replace('\\', '\\\\') return '"%s"'% s def flagsplit(s): """Converts a string of IMAP flags to a list :returns: E.g. '(\\Draft \\Deleted)' returns ['\\Draft','\\Deleted']. (FLAGS (\\Seen Old) UID 4807) returns ['FLAGS,'(\\Seen Old)','UID', '4807'] """ if s[0] != '(' or s[-1] != ')': raise ValueError("Passed s '%s' is not a flag list"% s) return imapsplit(s[1:-1]) def __options2hash(list): """convert list [1,2,3,4,5,6] to {1:2, 3:4, 5:6}""" # effectively this does dict(zip(l[::2],l[1::2])), however # measurements seemed to have indicated that the manual variant is # faster for mosly small lists. retval = {} counter = 0 while (counter < len(list)): retval[list[counter]] = list[counter + 1] counter += 2 __debug("__options2hash returning:", retval) return retval def flags2hash(flags): """Converts IMAP response string from eg IMAP4.fetch() to a hash. E.g. '(FLAGS (\\Seen Old) UID 4807)' leads to {'FLAGS': '(\\Seen Old)', 'UID': '4807'}""" return __options2hash(flagsplit(flags)) def imapsplit(imapstring): """Takes a string from an IMAP conversation and returns a list containing its components. One example string is: (\\HasNoChildren) "." "INBOX.Sent" The result from parsing this will be: ['(\\HasNoChildren)', '"."', '"INBOX.Sent"']""" if not isinstance(imapstring, str): __debug("imapsplit() got a non-string input; working around.") # Sometimes, imaplib will throw us a tuple if the input # contains a literal. See Python bug # #619732 at https://sourceforge.net/tracker/index.php?func=detail&aid=619732&group_id=5470&atid=105470 # One example is: # result[0] = '() "\\\\" Admin' # result[1] = ('() "\\\\" {19}', 'Folder\\2') # # This function will effectively get result[0] or result[1], so # if we get the result[1] version, we need to parse apart the tuple # and figure out what to do with it. Each even-numbered # part of it should end with the {} number, and each odd-numbered # part should be directly a part of the result. We'll # artificially quote it to help out. retval = [] for i in range(len(imapstring)): if i % 2: # Odd: quote then append. arg = imapstring[i] # Quote code lifted from imaplib arg = arg.replace('\\', '\\\\') arg = arg.replace('"', '\\"') arg = '"%s"' % arg __debug("imapsplit() non-string [%d]: Appending %s"% (i, arg)) retval.append(arg) else: # Even -- we have a string that ends with a literal # size specifier. We need to strip off that, then run # what remains through the regular imapsplit parser. # Recursion to the rescue. arg = imapstring[i] arg = re.sub('\{\d+\}$', '', arg) __debug("imapsplit() non-string [%d]: Feeding %s to recursion"%\ (i, arg)) retval.extend(imapsplit(arg)) __debug("imapsplit() non-string: returning %s" % str(retval)) return retval workstr = imapstring.strip() retval = [] while len(workstr): # handle parenthized fragments (...()...) if workstr[0] == '(': rparenc = 1 # count of right parenthesis to match rpareni = 1 # position to examine while rparenc: # Find the end of the group. if workstr[rpareni] == ')': # end of a group rparenc -= 1 elif workstr[rpareni] == '(': # start of a group rparenc += 1 rpareni += 1 # Move to next character. parenlist = workstr[0:rpareni] workstr = workstr[rpareni:].lstrip() retval.append(parenlist) elif workstr[0] == '"': # quoted fragments '"...\"..."' (quoted, rest) = __split_quoted(workstr) retval.append(quoted) workstr = rest else: splits = None # Python2 if hasattr(string, 'split'): splits = string.split(workstr, maxsplit = 1) # Python3 else: splits = str.split(workstr, maxsplit = 1) splitslen = len(splits) # The unquoted word is splits[0]; the remainder is splits[1] if splitslen == 2: # There's an unquoted word, and more string follows. retval.append(splits[0]) workstr = splits[1] # split will have already lstripped it continue elif splitslen == 1: # We got a last unquoted word, but nothing else retval.append(splits[0]) # Nothing remains. workstr would be '' break elif splitslen == 0: # There was not even an unquoted word. break return retval flagmap = [('\\Seen', 'S'), ('\\Answered', 'R'), ('\\Flagged', 'F'), ('\\Deleted', 'T'), ('\\Draft', 'D')] def flagsimap2maildir(flagstring): """Convert string '(\\Draft \\Deleted)' into a flags set(DR).""" retval = set() imapflaglist = flagstring[1:-1].split() for imapflag, maildirflag in flagmap: if imapflag in imapflaglist: retval.add(maildirflag) return retval def flagsimap2keywords(flagstring): """Convert string '(\\Draft \\Deleted somekeyword otherkeyword)' into a keyword set (somekeyword otherkeyword).""" imapflagset = set(flagstring[1:-1].split()) serverflagset = set([flag for (flag, c) in flagmap]) return imapflagset - serverflagset def flagsmaildir2imap(maildirflaglist): """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" retval = [] for imapflag, maildirflag in flagmap: if maildirflag in maildirflaglist: retval.append(imapflag) return '(' + ' '.join(sorted(retval)) + ')' def uid_sequence(uidlist): """Collapse UID lists into shorter sequence sets [1,2,3,4,5,10,12,13] will return "1:5,10,12:13". This function sorts the list, and only collapses if subsequent entries form a range. :returns: The collapsed UID list as string.""" def getrange(start, end): if start == end: return(str(start)) return "%s:%s"% (start, end) if not len(uidlist): return '' # Empty list, return start, end = None, None retval = [] # Force items to be longs and sort them sorted_uids = sorted(map(int, uidlist)) for item in iter(sorted_uids): item = int(item) if start == None: # First item start, end = item, item elif item == end + 1: # Next item in a range end = item else: # Starting a new range retval.append(getrange(start, end)) start, end = item, item retval.append(getrange(start, end)) # Add final range/item return ",".join(retval) def __split_quoted(s): """Looks for the ending quote character in the string that starts with quote character, splitting out quoted component and the rest of the string (without possible space between these two parts. First character of the string is taken to be quote character. Examples: - "this is \" a test" (\\None) => ("this is \" a test", (\\None)) - "\\" => ("\\", ) """ if len(s) == 0: return ('', '') q = quoted = s[0] rest = s[1:] while True: next_q = rest.find(q) if next_q == -1: raise ValueError("can't find ending quote '%s' in '%s'"% (q, s)) # If quote is preceeded by even number of backslashes, # then it is the ending quote, otherwise the quote # character is escaped by backslash, so we should # continue our search. is_escaped = False i = next_q - 1 while i >= 0 and rest[i] == '\\': i -= 1 is_escaped = not is_escaped quoted += rest[0:next_q + 1] rest = rest[next_q + 1:] if not is_escaped: return (quoted, rest.lstrip()) def format_labels_string(header, labels): """Formats labels for embedding into a message, with format according to header name. Headers from SPACE_SEPARATED_LABEL_HEADERS keep space-separated list of labels, the rest uses comma (',') as the separator. Also see parse_labels_string() and modify it accordingly if logics here gets changed.""" if header in SPACE_SEPARATED_LABEL_HEADERS: sep = ' ' else: sep = ',' return sep.join(labels) def parse_labels_string(header, labels_str): """Parses a string into a set of labels, with a format according to the name of the header. See __format_labels_string() for explanation on header handling and keep these two functions synced with each other. TODO: add test to ensure that - format_labels_string * parse_labels_string is unity and - parse_labels_string * format_labels_string is unity """ if header in SPACE_SEPARATED_LABEL_HEADERS: sep = ' ' else: sep = ',' labels = labels_str.strip().split(sep) return set([l.strip() for l in labels if l.strip()]) def labels_from_header(header_name, header_value): """Helper that builds label set from the corresponding header value. Arguments: - header_name: name of the header that keeps labels; - header_value: value of the said header, can be None Returns: set of labels parsed from the header (or empty set). """ if header_value: labels = parse_labels_string(header_name, header_value) else: labels = set() return labels def decode_mailbox_name(name): """Decodes a modified UTF-7 mailbox name. If the string cannot be decoded, it is returned unmodified. See RFC 3501, sec. 5.1.3. Arguments: - name: string, possibly encoded with modified UTF-7 Returns: decoded UTF-8 string. """ def demodify(m): s = m.group() if s == '+': return '+-' return '+' + s[1:-1].replace(',', '/') + '-' ret = MUTF7_SHIFT_RE.sub(demodify, name) try: return ret.decode('utf-7').encode('utf-8') except (UnicodeDecodeError, UnicodeEncodeError): return name # Functionality to convert folder names encoded in IMAP_utf_7 to utf_8. # This is achieved by defining 'imap4_utf_7' as a proper encoding scheme. # Public API, to be used in repository definitions def IMAP_utf8(foldername): """Convert IMAP4_utf_7 encoded string to utf-8""" return foldername.decode('imap4-utf-7').encode('utf-8') def utf8_IMAP(foldername): """Convert utf-8 encoded string to IMAP4_utf_7""" return foldername.decode('utf-8').encode('imap4-utf-7') # Codec definition def modified_base64(s): s = s.encode('utf-16be') return binascii.b2a_base64(s).rstrip('\n=').replace('/', ',') def doB64(_in, r): if _in: r.append('&%s-' % modified_base64(''.join(_in))) del _in[:] def encoder(s): r = [] _in = [] for c in s: ordC = ord(c) if 0x20 <= ordC <= 0x25 or 0x27 <= ordC <= 0x7e: doB64(_in, r) r.append(c) elif c == '&': doB64(_in, r) r.append('&-') else: _in.append(c) doB64(_in, r) return (str(''.join(r)), len(s)) # decoding def modified_unbase64(s): b = binascii.a2b_base64(s.replace(',', '/') + '===') return unicode(b, 'utf-16be') def decoder(s): r = [] decode = [] for c in s: if c == '&' and not decode: decode.append('&') elif c == '-' and decode: if len(decode) == 1: r.append('&') else: r.append(modified_unbase64(''.join(decode[1:]))) decode = [] elif decode: decode.append(c) else: r.append(c) if decode: r.append(modified_unbase64(''.join(decode[1:]))) bin_str = ''.join(r) return (bin_str, len(s)) class StreamReader(codecs.StreamReader): def decode(self, s, errors='strict'): return decoder(s) class StreamWriter(codecs.StreamWriter): def decode(self, s, errors='strict'): return encoder(s) def imap4_utf_7(name): if name == 'imap4-utf-7': return (encoder, decoder, StreamReader, StreamWriter) codecs.register(imap4_utf_7) ================================================ FILE: offlineimap/init.py ================================================ # OfflineIMAP initialization code # Copyright (C) 2002-2017 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import sys import threading import signal import socket import logging import traceback import collections from optparse import OptionParser import offlineimap import offlineimap.virtual_imaplib2 as imaplib # Ensure that `ui` gets loaded before `threadutil` in order to # break the circular dependency between `threadutil` and `Curses`. from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap import threadutil, accounts, folder, mbnames from offlineimap import globals as glob from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace from offlineimap.repository import Repository from offlineimap.folder.IMAP import MSGCOPY_NAMESPACE ACCOUNT_LIMITED_THREAD_NAME = 'MAX_ACCOUNTS' PYTHON_VERSION = sys.version.split(' ')[0] def syncitall(list_accounts, config): """The target when in multithreading mode for running accounts threads.""" threads = threadutil.accountThreads() # The collection of accounts threads. for accountname in list_accounts: # Start a new thread per account and store it in the collection. account = accounts.SyncableAccount(config, accountname) thread = threadutil.InstanceLimitedThread( ACCOUNT_LIMITED_THREAD_NAME, target = account.syncrunner, name = "Account sync %s"% accountname ) thread.setDaemon(True) # The add() method expects a started thread. thread.start() threads.add(thread) # Wait for the threads to finish. threads.wait() # Blocks until all accounts are processed. class OfflineImap(object): """The main class that encapsulates the high level use of OfflineImap. To invoke OfflineImap you would call it with:: oi = OfflineImap() oi.run() """ def get_env_info(self): info = "imaplib2 v%s (%s), Python v%s"% ( imaplib.__version__, imaplib.DESC, PYTHON_VERSION ) try: import ssl info = "%s, %s"% (info, ssl.OPENSSL_VERSION) except: pass return info def run(self): """Parse the commandline and invoke everything""" # next line also sets self.config and self.ui options, args = self.__parse_cmd_options() if options.diagnostics: self.__serverdiagnostics(options) elif options.migrate_fmd5: self.__migratefmd5(options) elif options.mbnames_prune: mbnames.init(self.config, self.ui, options.dryrun) mbnames.prune(self.config.get("general", "accounts")) mbnames.write() elif options.deletefolder: return self.__deletefolder(options) else: return self.__sync(options) def __parse_cmd_options(self): parser = OptionParser( version=offlineimap.__version__, description="%s.\n\n%s"% (offlineimap.__copyright__, offlineimap.__license__) ) parser.add_option("-V", action="store_true", dest="version", default=False, help="show full version infos") parser.add_option("--dry-run", action="store_true", dest="dryrun", default=False, help="dry run mode") parser.add_option("--info", action="store_true", dest="diagnostics", default=False, help="output information on the configured email repositories") parser.add_option("-1", action="store_true", dest="singlethreading", default=False, help="(the number one) disable all multithreading operations") parser.add_option("-P", dest="profiledir", metavar="DIR", help="sets OfflineIMAP into profile mode.") parser.add_option("-a", dest="accounts", metavar="account1[,account2[,...]]", help="list of accounts to sync") parser.add_option("-c", dest="configfile", metavar="FILE", default=None, help="specifies a configuration file to use") parser.add_option("-d", dest="debugtype", metavar="type1[,type2[,...]]", help="enables debugging for OfflineIMAP " " (types: imap, maildir, thread)") parser.add_option("-l", dest="logfile", metavar="FILE", help="log to FILE") parser.add_option("-s", action="store_true", dest="syslog", default=False, help="log to syslog") parser.add_option("-f", dest="folders", metavar="folder1[,folder2[,...]]", help="only sync the specified folders") parser.add_option("-k", dest="configoverride", action="append", metavar="[section:]option=value", help="override configuration file option") parser.add_option("-o", action="store_true", dest="runonce", default=False, help="run only once (ignore autorefresh)") parser.add_option("-q", action="store_true", dest="quick", default=False, help="run only quick synchronizations (don't update flags)") parser.add_option("-u", dest="interface", help="specifies an alternative user interface" " (quiet, basic, syslog, ttyui, blinkenlights, machineui)") parser.add_option("--delete-folder", dest="deletefolder", default=None, metavar="FOLDERNAME", help="Delete a folder (on the remote repository)") parser.add_option("--migrate-fmd5-using-nametrans", action="store_true", dest="migrate_fmd5", default=False, help="migrate FMD5 hashes from versions prior to 6.3.5") parser.add_option("--mbnames-prune", action="store_true", dest="mbnames_prune", default=False, help="remove mbnames entries for accounts not in accounts") (options, args) = parser.parse_args() glob.set_options(options) if options.version: print("offlineimap v%s, %s"% ( offlineimap.__version__, self.get_env_info()) ) sys.exit(0) # Read in configuration file. if not options.configfile: # Try XDG location, then fall back to ~/.offlineimaprc xdg_var = 'XDG_CONFIG_HOME' if not xdg_var in os.environ or not os.environ[xdg_var]: xdg_home = os.path.expanduser('~/.config') else: xdg_home = os.environ[xdg_var] options.configfile = os.path.join(xdg_home, "offlineimap", "config") if not os.path.exists(options.configfile): options.configfile = os.path.expanduser('~/.offlineimaprc') configfilename = options.configfile else: configfilename = os.path.expanduser(options.configfile) config = CustomConfigParser() if not os.path.exists(configfilename): # TODO, initialize and make use of chosen ui for logging logging.error(" *** Config file '%s' does not exist; aborting!"% configfilename) sys.exit(1) config.read(configfilename) # Profile mode chosen? if options.profiledir: if not options.singlethreading: # TODO, make use of chosen ui for logging logging.warn("Profile mode: Forcing to singlethreaded.") options.singlethreading = True if os.path.exists(options.profiledir): # TODO, make use of chosen ui for logging logging.warn("Profile mode: Directory '%s' already exists!"% options.profiledir) else: os.mkdir(options.profiledir) # TODO, make use of chosen ui for logging logging.warn("Profile mode: Potentially large data will be " "created in '%s'"% options.profiledir) # Override a config value. if options.configoverride: for option in options.configoverride: (key, value) = option.split('=', 1) if ':' in key: (secname, key) = key.split(':', 1) section = secname.replace("_", " ") else: section = "general" config.set(section, key, value) # Which ui to use? CLI option overrides config file. ui_type = config.getdefault('general', 'ui', 'ttyui') if options.interface != None: ui_type = options.interface if '.' in ui_type: # Transform Curses.Blinkenlights -> Blinkenlights. ui_type = ui_type.split('.')[-1] # TODO, make use of chosen ui for logging logging.warning('Using old interface name, consider using one ' 'of %s'% ', '.join(UI_LIST.keys())) if options.diagnostics: ui_type = 'ttyui' # Enforce this UI for --info. # dry-run? Set [general]dry-run=True. if options.dryrun: dryrun = config.set('general', 'dry-run', 'True') config.set_if_not_exists('general', 'dry-run', 'False') try: # Create the ui class. self.ui = UI_LIST[ui_type.lower()](config) except KeyError: logging.error("UI '%s' does not exist, choose one of: %s"% (ui_type, ', '.join(UI_LIST.keys()))) sys.exit(1) setglobalui(self.ui) # Set up additional log files. if options.logfile: self.ui.setlogfile(options.logfile) # Set up syslog. if options.syslog: self.ui.setup_sysloghandler() # Welcome blurb. self.ui.init_banner() self.ui.info(self.get_env_info()) if options.debugtype: self.ui.logger.setLevel(logging.DEBUG) if options.debugtype.lower() == 'all': options.debugtype = 'imap,maildir,thread' # Force single threading? if not ('thread' in options.debugtype.split(',') \ and not options.singlethreading): self.ui._msg("Debug mode: Forcing to singlethreaded.") options.singlethreading = True debugtypes = options.debugtype.split(',') + [''] for dtype in debugtypes: dtype = dtype.strip() self.ui.add_debug(dtype) if dtype.lower() == u'imap': imaplib.Debug = 5 if options.runonce: # Must kill the possible default option. if config.has_option('DEFAULT', 'autorefresh'): config.remove_option('DEFAULT', 'autorefresh') # FIXME: spaghetti code alert! for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") if options.quick: for section in accounts.getaccountlist(config): config.set('Account ' + section, "quick", '-1') # Custom folder list specified? if options.folders: foldernames = options.folders.split(",") folderfilter = "lambda f: f in %s"% foldernames folderincludes = "[]" for accountname in accounts.getaccountlist(config): account_section = 'Account ' + accountname remote_repo_section = 'Repository ' + \ config.get(account_section, 'remoterepository') config.set(remote_repo_section, "folderfilter", folderfilter) config.set(remote_repo_section, "folderincludes", folderincludes) if options.logfile: sys.stderr = self.ui.logfile socktimeout = config.getdefaultint("general", "socktimeout", 0) if socktimeout > 0: socket.setdefaulttimeout(socktimeout) threadutil.initInstanceLimit( ACCOUNT_LIMITED_THREAD_NAME, config.getdefaultint('general', 'maxsyncaccounts', 1) ) for reposname in config.getsectionlist('Repository'): # Limit the number of threads. Limitation on usage is handled at the # imapserver level. for namespace in [accounts.FOLDER_NAMESPACE + reposname, MSGCOPY_NAMESPACE + reposname]: if options.singlethreading: threadutil.initInstanceLimit(namespace, 1) else: threadutil.initInstanceLimit( namespace, config.getdefaultint( 'Repository ' + reposname, 'maxconnections', 2) ) self.config = config return (options, args) def __dumpstacks(self, context=1, sighandler_deep=2): """ Signal handler: dump a stack trace for each existing thread.""" currentThreadId = threading.currentThread().ident def unique_count(l): d = collections.defaultdict(lambda: 0) for v in l: d[tuple(v)] += 1 return list((k, v) for k, v in d.items()) stack_displays = [] for threadId, stack in sys._current_frames().items(): stack_display = [] for filename, lineno, name, line in traceback.extract_stack(stack): stack_display.append(' File: "%s", line %d, in %s' % (filename, lineno, name)) if line: stack_display.append(" %s" % (line.strip())) if currentThreadId == threadId: stack_display = stack_display[:- (sighandler_deep * 2)] stack_display.append(' => Stopped to handle current signal. ') stack_displays.append(stack_display) stacks = unique_count(stack_displays) self.ui.debug('thread', "** Thread List:\n") for stack, times in stacks: if times == 1: msg = "%s Thread is at:\n%s\n" else: msg = "%s Threads are at:\n%s\n" self.ui.debug('thread', msg % (times, '\n'.join(stack[- (context * 2):]))) self.ui.debug('thread', "Dumped a total of %d Threads." % len(sys._current_frames().keys())) def _get_activeaccounts(self, options): activeaccounts = [] errormsg = None activeaccountnames = self.config.get("general", "accounts") if options.accounts: activeaccountnames = options.accounts activeaccountnames = [x.lstrip() for x in activeaccountnames.split(",")] allaccounts = accounts.getaccountlist(self.config) for accountname in activeaccountnames: if accountname in allaccounts: activeaccounts.append(accountname) else: errormsg = "Valid accounts are: %s"% ( ", ".join(allaccounts)) self.ui.error("The account '%s' does not exist"% accountname) if len(activeaccounts) < 1: errormsg = "No accounts are defined!" if errormsg is not None: self.ui.terminate(1, errormsg=errormsg) return activeaccounts def __sync(self, options): """Invoke the correct single/multithread syncing self.config is supposed to have been correctly initialized already.""" def sig_handler(sig, frame): if sig == signal.SIGUSR1: # tell each account to stop sleeping accounts.Account.set_abort_event(self.config, 1) elif sig in (signal.SIGUSR2, signal.SIGABRT): # tell each account to stop looping getglobalui().warn("Terminating after this sync...") accounts.Account.set_abort_event(self.config, 2) elif sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): # tell each account to ABORT ASAP (ctrl-c) getglobalui().warn("Preparing to shutdown after sync (this may "\ "take some time), press CTRL-C three "\ "times to shutdown immediately") accounts.Account.set_abort_event(self.config, 3) if 'thread' in self.ui.debuglist: self.__dumpstacks(5) # Abort after three Ctrl-C keystrokes self.num_sigterm += 1 if self.num_sigterm >= 3: getglobalui().warn("Signaled thrice. Aborting!") sys.exit(1) elif sig == signal.SIGQUIT: stacktrace.dump(sys.stderr) os.abort() try: self.num_sigterm = 0 signal.signal(signal.SIGHUP, sig_handler) signal.signal(signal.SIGUSR1, sig_handler) signal.signal(signal.SIGUSR2, sig_handler) signal.signal(signal.SIGABRT, sig_handler) signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGQUIT, sig_handler) # Various initializations that need to be performed: activeaccounts = self._get_activeaccounts(options) mbnames.init(self.config, self.ui, options.dryrun) if options.singlethreading: # Singlethreaded. self.__sync_singlethreaded(activeaccounts, options.profiledir) else: # Multithreaded. t = threadutil.ExitNotifyThread( target=syncitall, name='Sync Runner', args=(activeaccounts, self.config,) ) # Special exit message for the monitor to stop looping. t.exit_message = threadutil.STOP_MONITOR t.start() threadutil.monitor() # All sync are done. mbnames.write() self.ui.terminate() return 0 except (SystemExit): raise except Exception as e: self.ui.error(e) self.ui.terminate() return 1 def __sync_singlethreaded(self, list_accounts, profiledir): """Executed in singlethreaded mode only. :param accs: A list of accounts that should be synced """ for accountname in list_accounts: account = accounts.SyncableAccount(self.config, accountname) threading.currentThread().name = \ "Account sync %s"% account.getname() if not profiledir: account.syncrunner() # Profile mode. else: try: import cProfile as profile except ImportError: import profile prof = profile.Profile() try: prof = prof.runctx("account.syncrunner()", globals(), locals()) except SystemExit: pass from datetime import datetime dt = datetime.now().strftime('%Y%m%d%H%M%S') prof.dump_stats(os.path.join( profiledir, "%s_%s.prof"% (dt, account.getname()))) def __serverdiagnostics(self, options): self.ui.info(" imaplib2: %s (%s)"% (imaplib.__version__, imaplib.DESC)) for accountname in self._get_activeaccounts(options): account = accounts.Account(self.config, accountname) account.serverdiagnostics() def __deletefolder(self, options): list_accounts = self._get_activeaccounts(options) if len(list_accounts) != 1: self.ui.error("you must supply only one account with '-a'") return 1 account = accounts.Account(self.config, list_accounts.pop()) return account.deletefolder(options.deletefolder) def __migratefmd5(self, options): for accountname in self._get_activeaccounts(options): account = accounts.Account(self.config, accountname) localrepo = Repository(account, 'local') if localrepo.getfoldertype() != folder.Maildir.MaildirFolder: continue folders = localrepo.getfolders() for f in folders: f.migratefmd5(options.dryrun) ================================================ FILE: offlineimap/localeval.py ================================================ """Eval python code with global namespace of a python source file.""" # Copyright (C) 2002-2016 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import imp try: import errno except: pass class LocalEval(object): """Here is a powerfull but very dangerous option, of course.""" def __init__(self, path=None): self.namespace = {} if path is not None: # FIXME: limit opening files owned by current user with rights set # to fixed mode 644. foo = open(path, 'r') module = imp.load_module( '', foo, path, ('', 'r', imp.PY_SOURCE)) for attr in dir(module): self.namespace[attr] = getattr(module, attr) def eval(self, text, namespace=None): names = {} names.update(self.namespace) if namespace is not None: names.update(namespace) return eval(text, names) ================================================ FILE: offlineimap/mbnames.py ================================================ # Mailbox name generator # Copyright (C) 2002-2016 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import codecs import re # For folderfilter. import json from threading import Lock from os import listdir, makedirs, path, unlink from sys import exc_info try: from ConfigParser import NoSectionError except ImportError: # Py3. from configparser import NoSectionError _mbLock = Lock() _mbnames = None # Called at sync time for each folder. def add(accountname, folder_root, foldername): global _mbnames if _mbnames.is_enabled() is not True: return with _mbLock: _mbnames.addAccountFolder(accountname, folder_root, foldername) # Called once. def init(conf, ui, dry_run): global _mbnames if _mbnames is None: _mbnames = _Mbnames(conf, ui, dry_run) # Called once. def prune(accounts): global _mbnames if _mbnames.is_enabled() is True: _mbnames.prune(accounts) else: _mbnames.pruneAll() # Called once. def write(): """Write the mbnames file.""" global _mbnames if _mbnames.is_enabled() is not True: return if _mbnames.get_incremental() is not True: _mbnames.write() # Called as soon as all the folders are synced for the account. def writeIntermediateFile(accountname): """Write intermediate mbnames file.""" global _mbnames if _mbnames.is_enabled() is not True: return _mbnames.writeIntermediateFile(accountname) if _mbnames.get_incremental() is True: _mbnames.write() class _IntermediateMbnames(object): """mbnames data for one account.""" def __init__(self, accountname, folder_root, mbnamesdir, folderfilter, dry_run, ui): self.ui = ui self._foldernames = [] self._accountname = accountname self._folder_root = folder_root self._folderfilter = folderfilter self._path = path.join(mbnamesdir, "%s.json"% accountname) self._dryrun = dry_run def add(self, foldername): if foldername not in self._foldernames: self._foldernames.append(foldername) def get_folder_root(self): return self._folder_root def write(self): """Write intermediate mbnames file in JSON format.""" itemlist = [] for foldername in self._foldernames: if self._folderfilter(self._accountname, foldername): itemlist.append({ 'accountname': self._accountname, 'foldername': foldername.decode('utf-8'), 'localfolders': self._folder_root, }) if self._dryrun: self.ui.info("mbnames would write %s"% self._path) else: with codecs.open( self._path, "wt", encoding='UTF-8') as intermediateFD: json.dump(itemlist, intermediateFD) class _Mbnames(object): def __init__(self, config, ui, dry_run): self._config = config self.ui = ui self._dryrun = dry_run self._enabled = None # Keys: accountname, values: _IntermediateMbnames instance. self._intermediates = {} self._incremental = None self._mbnamesdir = None self._path = None self._folderfilter = lambda accountname, foldername: True self._func_sortkey = lambda d: (d['accountname'], d['foldername']) localeval = config.getlocaleval() mbnamesdir = path.join(config.getmetadatadir(), "mbnames") self._peritem = None self._header = None self._sep = None self._footer = None try: if not self._dryrun: makedirs(mbnamesdir) except OSError: pass self._mbnamesdir = mbnamesdir try: self._enabled = self._config.getdefaultboolean( "mbnames", "enabled", False) self._peritem = self._config.get("mbnames", "peritem", raw=1) self._header = localeval.eval(config.get("mbnames", "header")) self._sep = localeval.eval(config.get("mbnames", "sep")) self._footer = localeval.eval(config.get("mbnames", "footer")) xforms = [path.expanduser, path.expandvars] self._path = config.apply_xforms( config.get("mbnames", "filename"), xforms) if self._config.has_option("mbnames", "sort_keyfunc"): self._func_sortkey = localeval.eval( self._config.get("mbnames", "sort_keyfunc"), {'re': re}) if self._config.has_option("mbnames", "folderfilter"): self._folderfilter = localeval.eval( self._config.get("mbnames", "folderfilter"), {'re': re}) except NoSectionError: pass def _iterIntermediateFiles(self): for foo in listdir(self._mbnamesdir): foo = path.join(self._mbnamesdir, foo) if path.isfile(foo) and foo[-5:] == '.json': yield foo def _removeIntermediateFile(self, path): if self._dryrun: self.ui.info("mbnames would remove %s"% path) else: unlink(path) self.ui.info("removed %s"% path) def addAccountFolder(self, accountname, folder_root, foldername): """Add foldername entry for an account.""" if accountname not in self._intermediates: self._intermediates[accountname] = _IntermediateMbnames( accountname, folder_root, self._mbnamesdir, self._folderfilter, self._dryrun, self.ui, ) self._intermediates[accountname].add(foldername) def get_incremental(self): if self._incremental is None: self._incremental = self._config.getdefaultboolean( "mbnames", "incremental", False) return self._incremental def is_enabled(self): return self._enabled def prune(self, accounts): removals = False for intermediateFile in self._iterIntermediateFiles(): filename = path.basename(intermediateFile) accountname = filename[:-5] if accountname not in accounts: removals = True self._removeIntermediateFile(intermediateFile) if removals is False: self.ui.info("no cache file to remove") def pruneAll(self): for intermediateFile in self._iterIntermediateFiles(): self._removeIntermediateFile(intermediateFile) def write(self): itemlist = [] for intermediateFile in self._iterIntermediateFiles(): try: with codecs.open( intermediateFile, 'rt', encoding="UTF-8") as intermediateFD: for item in json.load(intermediateFD): itemlist.append(item) except (OSError, IOError) as e: self.ui.error("could not read intermediate mbnames file '%s':" "%s"% (intermediateFile, str(e))) except Exception as e: self.ui.error( e, exc_info()[2], ("intermediate mbnames file %s not properly read"% intermediateFile) ) itemlist.sort(key=self._func_sortkey) itemlist = [self._peritem % d for d in itemlist] if self._dryrun: self.ui.info("mbnames would write %s"% self._path) else: try: with codecs.open( self._path, 'wt', encoding='UTF-8') as mbnamesFile: mbnamesFile.write(self._header) mbnamesFile.write(self._sep.join(itemlist)) mbnamesFile.write(self._footer) except (OSError, IOError) as e: self.ui.error( e, exc_info()[2], "mbnames file %s not properly written"% self._path ) def writeIntermediateFile(self, accountname): try: self._intermediates[accountname].write() except (OSError, IOError) as e: self.ui.error( e, exc_info()[2], "intermediate mbnames file %s not properly written"% self._path ) ================================================ FILE: offlineimap/repository/Base.py ================================================ """ Base repository support """ # Copyright (C) 2002-2017 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import re import os.path from sys import exc_info from offlineimap import CustomConfig from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError class BaseRepository(CustomConfig.ConfigHelperMixin): def __init__(self, reposname, account): self.ui = getglobalui() self.account = account self.config = account.getconfig() self.name = reposname self.localeval = account.getlocaleval() self._accountname = self.account.getname() self._readonly = self.getconfboolean('readonly', False) self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name) if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0o700) self.mapdir = os.path.join(self.uiddir, 'UIDMapping') if not os.path.exists(self.mapdir): os.mkdir(self.mapdir, 0o700) # FIXME: self.uiddir variable name is lying about itself. self.uiddir = os.path.join(self.uiddir, 'FolderValidity') if not os.path.exists(self.uiddir): os.mkdir(self.uiddir, 0o700) self.nametrans = lambda foldername: foldername self.folderfilter = lambda foldername: 1 self.folderincludes = [] self.foldersort = None self.newmail_hook = None if self.config.has_option(self.getsection(), 'nametrans'): self.nametrans = self.localeval.eval( self.getconf('nametrans'), {'re': re}) if self.config.has_option(self.getsection(), 'folderfilter'): self.folderfilter = self.localeval.eval( self.getconf('folderfilter'), {'re': re}) if self.config.has_option(self.getsection(), 'folderincludes'): self.folderincludes = self.localeval.eval( self.getconf('folderincludes'), {'re': re}) if self.config.has_option(self.getsection(), 'foldersort'): self.foldersort = self.localeval.eval( self.getconf('foldersort'), {'re': re}) def restore_atime(self): """Sets folders' atime back to their values after a sync Controlled by the 'restoreatime' config parameter (default False), applies only to local Maildir mailboxes and does nothing on all other repository types.""" pass def connect(self): """Establish a connection to the remote, if necessary. This exists so that IMAP connections can all be established up front, gathering passwords as needed. It was added in order to support the error recovery -- we need to connect first outside of the error trap in order to validate the password, and that's the point of this function.""" pass def holdordropconnections(self): pass def dropconnections(self): pass def getaccount(self): return self.account def getname(self): return self.name def __str__(self): return self.name @property def accountname(self): """Account name as string""" return self._accountname def getuiddir(self): return self.uiddir def getmapdir(self): return self.mapdir # Interface from CustomConfig.ConfigHelperMixin def getsection(self): return 'Repository ' + self.name # Interface from CustomConfig.ConfigHelperMixin def getconfig(self): return self.config @property def readonly(self): """Is the repository readonly?""" return self._readonly def getlocaleval(self): return self.account.getlocaleval() def getfolders(self): """Returns a list of ALL folders on this server.""" return [] def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" pass def getsep(self): raise NotImplementedError def getkeywordmap(self): raise NotImplementedError def should_sync_folder(self, fname): """Should this folder be synced?""" return fname in self.folderincludes or self.folderfilter(fname) def should_create_folders(self): """Is folder creation enabled on this repository? It is disabled by either setting the whole repository 'readonly' or by using the 'createfolders' setting.""" return (not self._readonly) and \ self.getconfboolean('createfolders', True) def makefolder(self, foldername): """Create a new folder.""" raise NotImplementedError def deletefolder(self, foldername): raise NotImplementedError def getfolder(self, foldername, decode=True): """Get the folder for this repo. WARNING: the signature changes whether it's remote or local: - remote types have the decode arg - local types don't have the decode arg """ raise NotImplementedError def sync_folder_structure(self, local_repo, status_repo): """Sync the folders structure. It does NOT sync the contents of those folders. nametrans rules in both directions will be honored Configuring nametrans on BOTH repositories could lead to infinite folder creation cycles.""" if not self.should_create_folders() and not local_repo.should_create_folders(): # Quick exit if no folder creation is enabled on either side. return remote_repo = self remote_hash, local_hash = {}, {} for folder in remote_repo.getfolders(): remote_hash[folder.getname()] = folder for folder in local_repo.getfolders(): local_hash[folder.getname()] = folder # Create new folders from remote to local. for remote_name, remote_folder in remote_hash.items(): # Don't create on local_repo, if it is readonly. if not local_repo.should_create_folders(): break # Apply remote nametrans and fix serparator. local_name = remote_folder.getvisiblename().replace( remote_repo.getsep(), local_repo.getsep()) if remote_folder.sync_this and not local_name in local_hash.keys(): try: local_repo.makefolder(local_name) # Need to refresh list. local_repo.forgetfolders() except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on repository %s"% (local_name, local_repo)) raise status_repo.makefolder(local_name.replace( local_repo.getsep(), status_repo.getsep())) # Create new folders from local to remote. for local_name, local_folder in local_hash.items(): if not remote_repo.should_create_folders(): # Don't create missing folder on readonly repo. break # Apply reverse nametrans and fix serparator. remote_name = local_folder.getvisiblename().replace( local_repo.getsep(), remote_repo.getsep()) if local_folder.sync_this and not remote_name in remote_hash.keys(): # Would the remote filter out the new folder name? In this case # don't create it. if not remote_repo.should_sync_folder(remote_name): self.ui.debug('', "Not creating folder '%s' (repository '%s" "') as it would be filtered out on that repository."% (remote_name, self)) continue # nametrans sanity check! Does remote nametrans lead to the # original local name? # # Apply remote nametrans to see if we end up with the same # name. We have: # - remote_name: local_name -> reverse nametrans + separator # We want local_name == loop_name from: # - remote_name -> remote (nametrans + separator) -> loop_name # # Get IMAPFolder and see if the reverse nametrans works fine. # TODO: getfolder() works only because we succeed in getting # inexisting folders which I would like to change. Take care! tmp_remotefolder = remote_repo.getfolder(remote_name, decode=False) loop_name = tmp_remotefolder.getvisiblename().replace( remote_repo.getsep(), local_repo.getsep()) if local_name != loop_name: raise OfflineImapError("INFINITE FOLDER CREATION DETECTED! " "Folder '%s' (repository '%s') would be created as fold" "er '%s' (repository '%s'). The latter becomes '%s' in " "return, leading to infinite folder creation cycles.\n " "SOLUTION: 1) Do set your nametrans rules on both repos" "itories so they lead to identical names if applied bac" "k and forth. 2) Use folderfilter settings on a reposit" "ory to prevent some folders from being created on the " "other side."% (local_folder.getname(), local_repo, remote_name, remote_repo, loop_name), OfflineImapError.ERROR.REPO) # End sanity check, actually create the folder. try: remote_repo.makefolder(remote_name) # Need to refresh list. self.forgetfolders() except OfflineImapError as e: self.ui.error(e, exc_info()[2], "Creating folder %s on " "repository %s"% (remote_name, remote_repo)) raise status_repo.makefolder(local_name.replace( local_repo.getsep(), status_repo.getsep())) # Find deleted folders. # TODO: We don't delete folders right now. return None def startkeepalive(self): """The default implementation will do nothing.""" pass def stopkeepalive(self): """Stop keep alive, but don't bother waiting for the threads to terminate.""" pass def getlocalroot(self): """ Local root folder for storing messages. Will not be set for remote repositories.""" return None ================================================ FILE: offlineimap/repository/Gmail.py ================================================ # Gmail IMAP repository support # Copyright (C) 2008-2016 Riccardo Murri & # contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.repository.IMAP import IMAPRepository from offlineimap import folder, OfflineImapError class GmailRepository(IMAPRepository): """Gmail IMAP repository. This class just has default settings for GMail's IMAP service. So you can do 'type = Gmail' instead of 'type = IMAP' and skip specifying the hostname, port etc. See http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 for the values we use.""" def __init__(self, reposname, account): """Initialize a GmailRepository object.""" IMAPRepository.__init__(self, reposname, account) def gethost(self): """Return the server name to connect to. We first check the usual IMAP settings, and then fall back to imap.gmail.com if nothing is specified.""" try: return super(GmailRepository, self).gethost() except OfflineImapError: # Nothing was configured, cache and return hardcoded # one. See the parent class (IMAPRepository) for how this # cache is used. self._host = "imap.gmail.com" return self._host def getoauth2_request_url(self): """Return the OAuth URL to request tokens from. We first check the usual OAuth settings, and then fall back to https://accounts.google.com/o/oauth2/token if nothing is specified.""" url = super(GmailRepository, self).getoauth2_request_url() if url is None: # Nothing was configured, cache and return hardcoded one. self.setoauth2_request_url("https://accounts.google.com/o/oauth2/token") else: self.setoauth2_request_url(url) return self.oauth2_request_url def getport(self): """Return the port number to connect to. This Gmail implementation first checks for the usual IMAP settings and falls back to 993 if nothing is specified.""" port = super(GmailRepository, self).getport() if port is None: return 993 else: return port def getssl(self): ssl = self.getconfboolean('ssl', None) if ssl is None: # Nothing was configured, return our default setting for # GMail. Maybe this should look more similar to gethost & # we could just rely on the global "ssl = yes" default. return True else: return ssl def getpreauthtunnel(self): return None def getfolder(self, foldername, decode=True): return self.getfoldertype()(self.imapserver, foldername, self, decode) def getfoldertype(self): return folder.Gmail.GmailFolder def gettrashfolder(self, foldername): # Where deleted mail should be moved return self.getconf('trashfolder', '[Gmail]/Trash') def getspamfolder(self): # Depending on the IMAP settings (Settings -> Forwarding and # POP/IMAP -> IMAP Access -> "When I mark a message in IMAP as # deleted") GMail might also deletes messages upon EXPUNGE in # the Spam folder. return self.getconf('spamfolder', '[Gmail]/Spam') ================================================ FILE: offlineimap/repository/GmailMaildir.py ================================================ # Maildir repository support # Copyright (C) 2002-2015 John Goerzen & contributors # # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.repository.Maildir import MaildirRepository from offlineimap.folder.GmailMaildir import GmailMaildirFolder class GmailMaildirRepository(MaildirRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" super(GmailMaildirRepository, self).__init__(reposname, account) def getfoldertype(self): return GmailMaildirFolder ================================================ FILE: offlineimap/repository/IMAP.py ================================================ """ IMAP repository support """ # Copyright (C) 2002-2019 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import netrc import errno import codecs from sys import exc_info from threading import Event import six from offlineimap import folder, imaputil, imapserver, OfflineImapError from offlineimap.repository.Base import BaseRepository from offlineimap.threadutil import ExitNotifyThread from offlineimap.utils.distro import get_os_sslcertfile, get_os_sslcertfile_searchpath class IMAPRepository(BaseRepository): def __init__(self, reposname, account): self.idlefolders = None BaseRepository.__init__(self, reposname, account) # self.ui is being set by the BaseRepository self._host = None # Must be set before calling imapserver.IMAPServer(self) self.oauth2_request_url = None self.imapserver = imapserver.IMAPServer(self) self.folders = None self.copy_ignore_eval = None # Keep alive. self.kaevent = None self.kathread = None # Only set the newmail_hook in an IMAP repository. if self.config.has_option(self.getsection(), 'newmail_hook'): self.newmail_hook = self.localeval.eval( self.getconf('newmail_hook')) if self.getconf('sep', None): self.ui.info("The 'sep' setting is being ignored for IMAP " "repository '%s' (it's autodetected)"% self) def startkeepalive(self): keepalivetime = self.getkeepalive() if not keepalivetime: return self.kaevent = Event() self.kathread = ExitNotifyThread(target=self.imapserver.keepalive, name="Keep alive " + self.getname(), args=(keepalivetime, self.kaevent)) self.kathread.setDaemon(1) self.kathread.start() def stopkeepalive(self): if self.kaevent is None: return # Keepalive is not active. self.kaevent.set() self.kathread = None self.kaevent = None def holdordropconnections(self): if not self.getholdconnectionopen(): self.dropconnections() def dropconnections(self): self.imapserver.close() def get_copy_ignore_UIDs(self, foldername): """Return a list of UIDs to not copy for this foldername.""" if self.copy_ignore_eval is None: if self.config.has_option(self.getsection(), 'copy_ignore_eval'): self.copy_ignore_eval = self.localeval.eval( self.getconf('copy_ignore_eval')) else: self.copy_ignore_eval = lambda x: None return self.copy_ignore_eval(foldername) def getholdconnectionopen(self): if self.getidlefolders(): return True return self.getconfboolean("holdconnectionopen", False) def getkeepalive(self): num = self.getconfint("keepalive", 0) if num == 0 and self.getidlefolders(): return 29*60 return num def getsep(self): """Return the folder separator for the IMAP repository This requires that self.imapserver has been initialized with an acquireconnection() or it will still be `None`""" assert self.imapserver.delim != None, "'%s' " \ "repository called getsep() before the folder separator was " \ "queried from the server"% self return self.imapserver.delim def gethost(self): """Return the configured hostname to connect to :returns: hostname as string or throws Exception""" if self._host: # Use cached value if possible. return self._host # 1) Check for remotehosteval setting. if self.config.has_option(self.getsection(), 'remotehosteval'): host = self.getconf('remotehosteval') try: host = self.localeval.eval(host) except Exception as e: six.reraise(OfflineImapError, OfflineImapError( "remotehosteval option for repository " "'%s' failed:\n%s"% (self, e), OfflineImapError.ERROR.REPO), exc_info()[2]) if host: self._host = host return self._host # 2) Check for plain remotehost setting. host = self.getconf('remotehost', None) if host != None: self._host = host return self._host # No success. raise OfflineImapError("No remote host for repository " "'%s' specified."% self, OfflineImapError.ERROR.REPO) def get_remote_identity(self): """Remote identity is used for certain SASL mechanisms (currently -- PLAIN) to inform server about the ID we want to authorize as instead of our login name.""" identity = self.getconf('remote_identity', default=None) if identity != None: identity = identity.encode('UTF-8') return identity def get_auth_mechanisms(self): supported = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] # Mechanisms are ranged from the strongest to the # weakest ones. # TODO: we need DIGEST-MD5, it must come before CRAM-MD5 # due to the chosen-plaintext resistance. default = ["GSSAPI", "XOAUTH2", "CRAM-MD5", "PLAIN", "LOGIN"] mechs = self.getconflist('auth_mechanisms', r',\s*', default) for m in mechs: if m not in supported: raise OfflineImapError("Repository %s: "% self + \ "unknown authentication mechanism '%s'"% m, OfflineImapError.ERROR.REPO) self.ui.debug('imap', "Using authentication mechanisms %s" % mechs) return mechs def getuser(self): user = None localeval = self.localeval if self.config.has_option(self.getsection(), 'remoteusereval'): user = self.getconf('remoteusereval') if user != None: return localeval.eval(user).encode('UTF-8') if self.config.has_option(self.getsection(), 'remoteuser'): # Assume the configuration file to be UTF-8 encoded so we must not # encode this string again. user = self.getconf('remoteuser') if user != None: return user try: netrcentry = netrc.netrc().authenticators(self.gethost()) except IOError as inst: if inst.errno != errno.ENOENT: raise else: if netrcentry: return netrcentry[0] try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: return netrcentry[0] def getport(self): port = None if self.config.has_option(self.getsection(), 'remoteporteval'): port = self.getconf('remoteporteval') if port != None: return self.localeval.eval(port) return self.getconfint('remoteport', None) def getipv6(self): return self.getconfboolean('ipv6', None) def getssl(self): return self.getconfboolean('ssl', True) def getsslclientcert(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] return self.getconf_xform('sslclientcert', xforms, None) def getsslclientkey(self): xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] return self.getconf_xform('sslclientkey', xforms, None) def getsslcacertfile(self): """Determines CA bundle. Returns path to the CA bundle. It is either explicitely specified or requested via "OS-DEFAULT" value (and we will search known locations for the current OS and distribution). If search via "OS-DEFAULT" route yields nothing, we will throw an exception to make our callers distinguish between not specified value and non-existent default CA bundle. It is also an error to specify non-existent file via configuration: it will error out later, but, perhaps, with less verbose explanation, so we will also throw an exception. It is consistent with the above behaviour, so any explicitely-requested configuration that doesn't result in an existing file will give an exception. """ xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath] cacertfile = self.getconf_xform('sslcacertfile', xforms, None) # Can't use above cacertfile because of abspath. if self.getconf('sslcacertfile', None) == "OS-DEFAULT": cacertfile = get_os_sslcertfile() if cacertfile == None: searchpath = get_os_sslcertfile_searchpath() if searchpath: reason = "Default CA bundle was requested, "\ "but no existing locations available. "\ "Tried %s." % (", ".join(searchpath)) else: reason = "Default CA bundle was requested, "\ "but OfflineIMAP doesn't know any for your "\ "current operating system." raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) if cacertfile is None: return None if not os.path.isfile(cacertfile): reason = "CA certfile for repository '%s' couldn't be found. "\ "No such file: '%s'" % (self.name, cacertfile) raise OfflineImapError(reason, OfflineImapError.ERROR.REPO) return cacertfile def gettlslevel(self): return self.getconf('tls_level', 'tls_compat') def getsslversion(self): return self.getconf('ssl_version', None) def getstarttls(self): return self.getconfboolean('starttls', True) def get_ssl_fingerprint(self): """Return array of possible certificate fingerprints. Configuration item cert_fingerprint can contain multiple comma-separated fingerprints in hex form.""" value = self.getconf('cert_fingerprint', "") return [f.strip().lower().replace(":", "") for f in value.split(',') if f] def setoauth2_request_url(self, url): self.oauth2_request_url = url def getoauth2_request_url(self): if self.oauth2_request_url is not None: # Use cached value if possible. return self.oauth2_request_url self.setoauth2_request_url(self.getconf('oauth2_request_url', None)) return self.oauth2_request_url def getoauth2_refresh_token(self): refresh_token = self.getconf('oauth2_refresh_token', None) if refresh_token is None: refresh_token = self.localeval.eval( self.getconf('oauth2_refresh_token_eval', "None") ) if refresh_token is not None: refresh_token = refresh_token.strip("\n") return refresh_token def getoauth2_access_token(self): access_token = self.getconf('oauth2_access_token', None) if access_token is None: access_token = self.localeval.eval( self.getconf('oauth2_access_token_eval', "None") ) if access_token is not None: access_token = access_token.strip("\n") return access_token def getoauth2_client_id(self): client_id = self.getconf('oauth2_client_id', None) if client_id is None: client_id = self.localeval.eval( self.getconf('oauth2_client_id_eval', "None") ) if client_id is not None: client_id = client_id.strip("\n") return client_id def getoauth2_client_secret(self): client_secret = self.getconf('oauth2_client_secret', None) if client_secret is None: client_secret = self.localeval.eval( self.getconf('oauth2_client_secret_eval', "None") ) if client_secret is not None: client_secret = client_secret.strip("\n") return client_secret def getpreauthtunnel(self): return self.getconf('preauthtunnel', None) def gettransporttunnel(self): return self.getconf('transporttunnel', None) def getreference(self): return self.getconf('reference', '') def getdecodefoldernames(self): return self.getconfboolean('decodefoldernames', False) def getidlefolders(self): if self.idlefolders is None: self.idlefolders = self.localeval.eval( self.getconf('idlefolders', '[]') ) return self.idlefolders def getmaxconnections(self): num1 = len(self.getidlefolders()) num2 = self.getconfint('maxconnections', 1) return max(num1, num2) def getexpunge(self): return self.getconfboolean('expunge', True) def getpassword(self): """Return the IMAP password for this repository. It tries to get passwords in the following order: 1. evaluate Repository 'remotepasseval' 2. read password from Repository 'remotepass' 3. read password from file specified in Repository 'remotepassfile' 4. read password from ~/.netrc 5. read password from /etc/netrc On success we return the password. If all strategies fail we return None.""" # 1. Evaluate Repository 'remotepasseval'. passwd = self.getconf('remotepasseval', None) if passwd is not None: return self.localeval.eval(passwd).encode('UTF-8') # 2. Read password from Repository 'remotepass'. password = self.getconf('remotepass', None) if password is not None: # Assume the configuration file to be UTF-8 encoded so we must not # encode this string again. return password # 3. Read password from file specified in Repository 'remotepassfile'. passfile = self.getconf('remotepassfile', None) if passfile is not None: fd = codecs.open(os.path.expanduser(passfile), 'r', 'UTF-8') password = fd.readline().strip() fd.close() return password.encode('UTF-8') # 4. Read password from ~/.netrc. try: netrcentry = netrc.netrc().authenticators(self.gethost()) except IOError as inst: if inst.errno != errno.ENOENT: raise else: if netrcentry: user = self.getuser() if user is None or user == netrcentry[0]: return netrcentry[2] # 5. Read password from /etc/netrc. try: netrcentry = netrc.netrc('/etc/netrc').authenticators(self.gethost()) except IOError as inst: if inst.errno not in (errno.ENOENT, errno.EACCES): raise else: if netrcentry: user = self.getuser() if user is None or user == netrcentry[0]: return netrcentry[2] # No strategy yielded a password! return None def getfolder(self, foldername, decode=True): """Return instance of OfflineIMAP representative folder.""" return self.getfoldertype()(self.imapserver, foldername, self, decode) def getfoldertype(self): return folder.IMAP.IMAPFolder def connect(self): imapobj = self.imapserver.acquireconnection() self.imapserver.releaseconnection(imapobj) def forgetfolders(self): self.folders = None def getfolders(self): """Return a list of instances of OfflineIMAP representative folder.""" if self.folders is not None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: result, listresult = listfunction(directory=self.imapserver.reference) if result != 'OK': raise OfflineImapError("Could not list the folders for" " repository %s. Server responded: %s"% (self.name, self, str(listresult)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) for s in listresult: if s == None or \ (isinstance(s, str) and s == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue try: flags, delim, name = imaputil.imapsplit(s) except ValueError: self.ui.error( "could not correctly parse server response; got: %s"% s) raise flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue retval.append(self.getfoldertype()(self.imapserver, name, self)) # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()( self.imapserver, foldername, self, decode=False)) finally: self.imapserver.releaseconnection(imapobj) if self.foldersort is None: # default sorting by case insensitive transposed name retval.sort(key=lambda x: str.lower(x.getvisiblename())) else: # do foldersort in a python3-compatible way # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function def cmp2key(mycmp): """Converts a cmp= function into a key= function We need to keep cmp functions for backward compatibility""" class K(object): def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) return K retval.sort(key=cmp2key(self.foldersort)) self.folders = retval return self.folders def deletefolder(self, foldername): """Delete a folder on the IMAP server.""" if self.account.utf_8_support: foldername = imaputil.utf8_IMAP(foldername) imapobj = self.imapserver.acquireconnection() try: result = imapobj.delete(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be deleted. " "Server responded: %s"% (foldername, self, str(result)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) def makefolder(self, foldername): """Create a folder on the IMAP server This will not update the list cached in :meth:`getfolders`. You will need to invoke :meth:`forgetfolders` to force new caching when you are done creating folders yourself. :param foldername: Full path of the folder to be created.""" if foldername is '': return if self.getreference(): foldername = self.getreference() + self.getsep() + foldername if not foldername: # Create top level folder as folder separator. foldername = self.getsep() self.makefolder_single(foldername) return parts = foldername.split(self.getsep()) folder_paths = [self.getsep().join(parts[:n + 1]) for n in range(len(parts))] for folder_path in folder_paths: try: self.makefolder_single(folder_path) except OfflineImapError as e: if '[ALREADYEXISTS]' not in e.reason: raise def makefolder_single(self, foldername): self.ui.makefolder(self, foldername) if self.account.dryrun: return imapobj = self.imapserver.acquireconnection() try: if self.account.utf_8_support: foldername = imaputil.utf8_IMAP(foldername) result = imapobj.create(foldername) if result[0] != 'OK': raise OfflineImapError("Folder '%s'[%s] could not be created. " "Server responded: %s"% (foldername, self, str(result)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) class MappedIMAPRepository(IMAPRepository): def getfoldertype(self): return folder.UIDMaps.MappedIMAPFolder ================================================ FILE: offlineimap/repository/LocalStatus.py ================================================ # Local status cache repository support # Copyright (C) 2002-2017 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder from offlineimap.repository.Base import BaseRepository from offlineimap.error import OfflineImapError class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) # class and root for all backends. self.backends = {} self.backends['sqlite'] = { 'class': LocalStatusSQLiteFolder, 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') } self.backends['plain'] = { 'class': LocalStatusFolder, 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') } if self.account.getconf('status_backend', None) is not None: raise OfflineImapError( "the 'status_backend' configuration option is not supported" " anymore; please, remove this configuration option.", OfflineImapError.ERROR.REPO ) # Set class and root for sqlite. self.setup_backend('sqlite') if not os.path.exists(self.root): os.mkdir(self.root, 0o700) # self._folders is a dict of name:LocalStatusFolders(). self._folders = {} def _instanciatefolder(self, foldername): return self.LocalStatusFolderClass(foldername, self) # Instanciate. def setup_backend(self, backend): if backend in self.backends.keys(): self._backend = backend self.root = self.backends[backend]['root'] self.LocalStatusFolderClass = self.backends[backend]['class'] def import_other_backend(self, folder): for bk, dic in self.backends.items(): # Skip folder's own type. if dic['class'] == type(folder): continue repobk = LocalStatusRepository(self.name, self.account) repobk.setup_backend(bk) # Fake the backend. folderbk = dic['class'](folder.name, repobk) # If backend contains data, import it to folder. if not folderbk.isnewfolder(): self.ui._msg("Migrating LocalStatus cache from %s to %s " "status folder for %s:%s"% (bk, self._backend, self.name, folder.name)) folderbk.cachemessagelist() folder.messagelist = folderbk.messagelist folder.saveall() break def getsep(self): return '.' def makefolder(self, foldername): """Create a LocalStatus Folder.""" if self.account.dryrun: return # Bail out in dry-run mode. # Create an empty StatusFolder. folder = self._instanciatefolder(foldername) # First delete any existing data to make sure we won't consider obsolete # data. This might happen if the user removed the folder (maildir) and # it is re-created afterwards. folder.purge() folder.openfiles() folder.save() folder.closefiles() # Invalidate the cache. self.forgetfolders() def getfolder(self, foldername): """Return the Folder() object for a foldername. Caller must call closefiles() on the folder when done.""" if foldername in self._folders: return self._folders[foldername] folder = self._instanciatefolder(foldername) # If folder is empty, try to import data from an other backend. if folder.isnewfolder(): self.import_other_backend(folder) self._folders[foldername] = folder return folder def getfolders(self): """Returns a list of all cached folders. Does nothing for this backend. We mangle the folder file names (see getfolderfilename) so we can not derive folder names from the file names that we have available. TODO: need to store a list of folder names somehow?""" pass def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" self._folders = {} ================================================ FILE: offlineimap/repository/Maildir.py ================================================ # Maildir repository support # Copyright (C) 2002-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap import folder from offlineimap.ui import getglobalui from offlineimap.error import OfflineImapError from offlineimap.repository.Base import BaseRepository import os from stat import * class MaildirRepository(BaseRepository): def __init__(self, reposname, account): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" BaseRepository.__init__(self, reposname, account) self.root = self.getlocalroot() self.folders = None self.ui = getglobalui() self.debug("MaildirRepository initialized, sep is %s"% repr(self.getsep())) self.folder_atimes = [] # Create the top-level folder if it doesn't exist if not os.path.isdir(self.root): os.makedirs(self.root, 0o700) # Create the keyword->char mapping self.keyword2char = dict() for c in 'abcdefghijklmnopqrstuvwxyz': confkey = 'customflag_' + c keyword = self.getconf(confkey, None) if keyword is not None: self.keyword2char[keyword] = c def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" p = os.path.join(self.root, foldername) new = os.path.join(p, 'new') cur = os.path.join(p, 'cur') atimes = (p, os.path.getatime(new), os.path.getatime(cur)) self.folder_atimes.append(atimes) def restore_atime(self): """Sets folders' atime back to their values after a sync Controlled by the 'restoreatime' config parameter.""" if not self.getconfboolean('restoreatime', False): return # not configured to restore for (dirpath, new_atime, cur_atime) in self.folder_atimes: new_dir = os.path.join(dirpath, 'new') cur_dir = os.path.join(dirpath, 'cur') os.utime(new_dir, (new_atime, os.path.getmtime(new_dir))) os.utime(cur_dir, (cur_atime, os.path.getmtime(cur_dir))) def getlocalroot(self): xforms = [os.path.expanduser, os.path.expandvars] return self.getconf_xform('localfolders', xforms) def debug(self, msg): self.ui.debug('maildir', msg) def getsep(self): return self.getconf('sep', '.').strip() def getkeywordmap(self): return self.keyword2char if len(self.keyword2char) > 0 else None def makefolder(self, foldername): """Create new Maildir folder if necessary This will not update the list cached in getfolders(). You will need to invoke :meth:`forgetfolders` to force new caching when you are done creating folders yourself. :param foldername: A relative mailbox name. The maildir will be created in self.root+'/'+foldername. All intermediate folder levels will be created if they do not exist yet. 'cur', 'tmp', and 'new' subfolders will be created in the maildir. """ self.ui.makefolder(self, foldername) if self.account.dryrun: return full_path = os.path.abspath(os.path.join(self.root, foldername)) # sanity tests if self.getsep() == '/': for component in foldername.split('/'): assert not component in ['new', 'cur', 'tmp'],\ "When using nested folders (/ as a Maildir separator), "\ "folder names may not contain 'new', 'cur', 'tmp'." assert foldername.find('../') == -1, "Folder names may not contain ../" assert not foldername.startswith('/'), "Folder names may not begin with /" # If we're using hierarchical folders, it's possible that # sub-folders may be created before higher-up ones. self.debug("makefolder: calling makedirs '%s'"% full_path) try: os.makedirs(full_path, 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already a directory"% foldername) else: raise for subdir in ['cur', 'new', 'tmp']: try: os.mkdir(os.path.join(full_path, subdir), 0o700) except OSError as e: if e.errno == 17 and os.path.isdir(full_path): self.debug("makefolder: '%s' already has subdir %s"% (foldername, subdir)) else: raise def deletefolder(self, foldername): self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s"% foldername) def getfolder(self, foldername): """Return a Folder instance of this Maildir If necessary, scan and cache all foldernames to make sure that we only return existing folders and that 2 calls with the same name will return the same object.""" # getfolders() will scan and cache the values *if* necessary folders = self.getfolders() for f in folders: if foldername == f.name: return f raise OfflineImapError("getfolder() asked for a nonexisting " "folder '%s'."% foldername, OfflineImapError.ERROR.FOLDER) def _getfolders_scandir(self, root, extension=None): """Recursively scan folder 'root'; return a list of MailDirFolder :param root: (absolute) path to Maildir root :param extension: (relative) subfolder to examine within root""" self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s"% (root, extension)) retval = [] # Configure the full path to this repository -- "toppath" if extension: toppath = os.path.join(root, extension) else: toppath = root self.debug(" toppath = %s"% toppath) # Iterate over directories in top & top itself. for dirname in os.listdir(toppath) + ['']: self.debug(" dirname = %s"% dirname) if dirname == '' and extension is not None: self.debug(' skip this entry (already scanned)') continue if dirname in ['cur', 'new', 'tmp']: self.debug(" skip this entry (Maildir special)") # Bypass special files. continue fullname = os.path.join(toppath, dirname) if not os.path.isdir(fullname): self.debug(" skip this entry (not a directory)") # Not a directory -- not a folder. continue # extension can be None. if extension: foldername = os.path.join(extension, dirname) else: foldername = dirname if (os.path.isdir(os.path.join(fullname, 'cur')) and os.path.isdir(os.path.join(fullname, 'new')) and os.path.isdir(os.path.join(fullname, 'tmp'))): # This directory has maildir stuff -- process self.debug(" This is maildir folder '%s'."% foldername) if self.getconfboolean('restoreatime', False): self._append_folder_atimes(foldername) fd = self.getfoldertype()(self.root, foldername, self.getsep(), self) retval.append(fd) if self.getsep() == '/' and dirname != '': # Recursively check sub-directories for folders too. retval.extend(self._getfolders_scandir(root, foldername)) self.debug("_GETFOLDERS_SCANDIR RETURNING %s"% \ repr([x.getname() for x in retval])) return retval def getfolders(self): if self.folders == None: self.folders = self._getfolders_scandir(self.root) return self.folders def getfoldertype(self): return folder.Maildir.MaildirFolder def forgetfolders(self): """Forgets the cached list of folders, if any. Useful to run after a sync run.""" self.folders = None ================================================ FILE: offlineimap/repository/__init__.py ================================================ # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import six from sys import exc_info try: from configparser import NoSectionError except ImportError: #python2 from ConfigParser import NoSectionError from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository from offlineimap.repository.Gmail import GmailRepository from offlineimap.repository.Maildir import MaildirRepository from offlineimap.repository.GmailMaildir import GmailMaildirRepository from offlineimap.repository.LocalStatus import LocalStatusRepository from offlineimap.error import OfflineImapError class Repository(object): """Abstract class that returns the correct Repository type instance based on 'account' and 'reqtype', e.g. a class:`ImapRepository` instance.""" def __new__(cls, account, reqtype): """ :param account: :class:`Account` :param regtype: 'remote', 'local', or 'status'""" if reqtype == 'remote': name = account.getconf('remoterepository') # We don't support Maildirs on the remote side. typemap = {'IMAP': IMAPRepository, 'Gmail': GmailRepository} elif reqtype == 'local': name = account.getconf('localrepository') typemap = {'IMAP': MappedIMAPRepository, 'Maildir': MaildirRepository, 'GmailMaildir': GmailMaildirRepository} elif reqtype == 'status': # create and return a LocalStatusRepository. name = account.getconf('localrepository') return LocalStatusRepository(name, account) else: errstr = "Repository type %s not supported" % reqtype raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO) # Get repository type. config = account.getconfig() try: repostype = config.get('Repository ' + name, 'type').strip() except NoSectionError as e: errstr = ("Could not find section '%s' in configuration. Required " "for account '%s'." % ('Repository %s' % name, account)) six.reraise(OfflineImapError, OfflineImapError(errstr, OfflineImapError.ERROR.REPO), exc_info()[2]) try: repo = typemap[repostype] except KeyError: errstr = "'%s' repository not supported for '%s' repositories."% \ (repostype, reqtype) six.reraise(OfflineImapError, OfflineImapError(errstr, OfflineImapError.ERROR.REPO), exc_info()[2]) return repo(name, account) def __init__(self, account, reqtype): """Load the correct Repository type and return that. The __init__ of the corresponding Repository class will be executed instead of this stub :param account: :class:`Account` :param regtype: 'remote', 'local', or 'status' """ pass ================================================ FILE: offlineimap/threadutil.py ================================================ # Copyright (C) 2002-2016 John Goerzen & contributors # Thread support module # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import Lock, Thread, BoundedSemaphore, currentThread try: from Queue import Queue, Empty except ImportError: # python3 from queue import Queue, Empty import traceback import os.path from offlineimap.ui import getglobalui STOP_MONITOR = 'STOP_MONITOR' ###################################################################### # General utilities ###################################################################### def semaphorereset(semaphore, originalstate): """Block until `semaphore` gets back to its original state, ie all acquired resources have been released.""" for i in range(originalstate): semaphore.acquire() # Now release these. for i in range(originalstate): semaphore.release() class accountThreads(object): """Store the list of all threads in the software so it can be used to find out what's running and what's not.""" def __init__(self): self.lock = Lock() self.list = [] def add(self, thread): with self.lock: self.list.append(thread) def remove(self, thread): with self.lock: self.list.remove(thread) def pop(self): with self.lock: if len(self.list) < 1: return None return self.list.pop() def wait(self): while True: thread = self.pop() if thread is None: break thread.join() ###################################################################### # Exit-notify threads ###################################################################### exitedThreads = Queue() def monitor(): """An infinite "monitoring" loop watching for finished ExitNotifyThread's. This one is supposed to run in the main thread. :param callback: the function to call when a thread terminated. That function is called with a single argument -- the ExitNotifyThread that has terminated. The monitor will not continue to monitor for other threads until 'callback' returns, so if it intends to perform long calculations, it should start a new thread itself -- but NOT an ExitNotifyThread, or else an infinite loop may result. Furthermore, the monitor will hold the lock all the while the other thread is waiting. :type callback: a callable function """ global exitedThreads ui = getglobalui() while True: # Loop forever and call 'callback' for each thread that exited try: # We need a timeout in the get() call, so that ctrl-c can throw a # SIGINT (http://bugs.python.org/issue1360). A timeout with empty # Queue will raise `Empty`. # # ExitNotifyThread add themselves to the exitedThreads queue once # they are done (normally or with exception). thread = exitedThreads.get(True, 60) # Request to abort when callback returns True. if thread.exit_exception is not None: if isinstance(thread.exit_exception, SystemExit): # Bring a SystemExit into the main thread. # Do not send it back to UI layer right now. # Maybe later send it to ui.terminate? raise SystemExit ui.threadException(thread) # Expected to terminate the program. # Should never hit this line. raise AssertionError("thread has 'exit_exception' set to" " '%s' [%s] but this value is unexpected" " and the ui did not stop the program."% (repr(thread.exit_exception), type(thread.exit_exception))) # Only the monitor thread has this exit message set. elif thread.exit_message == STOP_MONITOR: break # Exit the loop here. else: ui.threadExited(thread) except Empty: pass class ExitNotifyThread(Thread): """This class is designed to alert a "monitor" to the fact that a thread has exited and to provide for the ability for it to find out why. All instances are made daemon threads (setDaemon(True), so we bail out when the mainloop dies. The thread can set instance variables self.exit_message for a human readable reason of the thread exit. There is one instance of this class at runtime. The main thread waits for the monitor to end.""" def __init__(self, *args, **kwargs): super(ExitNotifyThread, self).__init__(*args, **kwargs) # These are all child threads that are supposed to go away when # the main thread is killed. self.setDaemon(True) self.exit_message = None self._exit_exc = None self._exit_stacktrace = None def run(self): """Allow profiling of a run and store exceptions.""" global exitedThreads try: Thread.run(self) except Exception as e: # Thread exited with Exception, store it tb = traceback.format_exc() self.set_exit_exception(e, tb) exitedThreads.put(self, True) def set_exit_exception(self, exc, st=None): """Sets Exception and stacktrace of a thread, so that other threads can query its exit status""" self._exit_exc = exc self._exit_stacktrace = st @property def exit_exception(self): """Returns the cause of the exit, one of: Exception() -- the thread aborted with this exception None -- normal termination.""" return self._exit_exc @property def exit_stacktrace(self): """Returns a string representing the stack trace if set""" return self._exit_stacktrace ###################################################################### # Instance-limited threads ###################################################################### limitedNamespaces = {} def initInstanceLimit(limitNamespace, instancemax): """Initialize the instance-limited thread implementation. Run up to intancemax threads for the given limitNamespace. This allows to honor maxsyncaccounts and maxconnections.""" global limitedNamespaces if not limitNamespace in limitedNamespaces: limitedNamespaces[limitNamespace] = BoundedSemaphore(instancemax) class InstanceLimitedThread(ExitNotifyThread): def __init__(self, limitNamespace, *args, **kwargs): self.limitNamespace = limitNamespace super(InstanceLimitedThread, self).__init__(*args, **kwargs) def start(self): global limitedNamespaces # Will block until the semaphore has free slots. limitedNamespaces[self.limitNamespace].acquire() ExitNotifyThread.start(self) def run(self): global limitedNamespaces try: ExitNotifyThread.run(self) finally: if limitedNamespaces and limitedNamespaces[self.limitNamespace]: limitedNamespaces[self.limitNamespace].release() ================================================ FILE: offlineimap/ui/Curses.py ================================================ # Curses-based interfaces # Copyright (C) 2003-2018 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import RLock, currentThread, Lock, Event from collections import deque import time import sys import os import curses import logging from offlineimap.ui.UIBase import UIBase from offlineimap.threadutil import ExitNotifyThread import offlineimap class CursesUtil: def __init__(self, *args, **kwargs): # iolock protects access to the self.iolock = RLock() self.tframe_lock = RLock() # tframe_lock protects the self.threadframes manipulation to # only happen from 1 thread. self.colormap = {} """dict, translating color string to curses color pair number""" def curses_colorpair(self, col_name): """Return the curses color pair, that corresponds to the color.""" return curses.color_pair(self.colormap[col_name]) def init_colorpairs(self): """Initialize the curses color pairs available.""" # set special colors 'gray' and 'banner' self.colormap['white'] = 0 #hardcoded by curses curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) self.colormap['banner'] = 1 # color 'banner' for bannerwin bcol = curses.COLOR_BLACK colors = ( # name, color, bold? ('black', curses.COLOR_BLACK, False), ('blue', curses.COLOR_BLUE,False), ('red', curses.COLOR_RED, False), ('purple', curses.COLOR_MAGENTA, False), ('cyan', curses.COLOR_CYAN, False), ('green', curses.COLOR_GREEN, False), ('orange', curses.COLOR_YELLOW, False)) #set the rest of all colors starting at pair 2 i = 1 for name, fcol, bold in colors: i += 1 self.colormap[name] = i curses.init_pair(i, fcol, bcol) def lock(self, block=True): """Locks the Curses ui thread. Can be invoked multiple times from the owning thread. Invoking from a non-owning thread blocks and waits until it has been unlocked by the owning thread.""" return self.iolock.acquire(block) def unlock(self): """Unlocks the Curses ui thread. Decrease the lock counter by one and unlock the ui thread if the counter reaches 0. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked.""" self.iolock.release() def exec_locked(self, target, *args, **kwargs): """Perform an operation with full locking.""" self.lock() try: target(*args, **kwargs) finally: self.unlock() def refresh(self): def lockedstuff(): curses.panel.update_panels() curses.doupdate() self.exec_locked(lockedstuff) def isactive(self): return hasattr(self, 'stdscr') class CursesAccountFrame: """Notable instance variables: - account: corresponding Account() - children - ui - key - window: curses window associated with an account """ def __init__(self, ui, account): """ :param account: An Account() or None (for eg SyncrunnerThread)""" self.children = [] self.account = account if account else '*Control' self.ui = ui self.window = None # Curses window associated with this acc. self.acc_num = None # Account number (& hotkey) associated with this acc. self.location = 0 # length of the account prefix string def drawleadstr(self, secs = 0): """Draw the account status string. secs tells us how long we are going to sleep.""" sleepstr = '%3d:%02d'% (secs // 60, secs % 60) if secs else 'active' accstr = '%s: [%s] %12.12s: '% (self.acc_num, sleepstr, self.account) def addstr(): try: self.window.addstr(0, 0, accstr) except curses.error as e: # Occurs when the terminal is very small pass self.ui.exec_locked(addstr); self.location = len(accstr) def setwindow(self, curses_win, acc_num): """Register an curses win and a hotkey as Account window. :param curses_win: the curses window associated with an account :param acc_num: int denoting the hotkey associated with this account.""" self.window = curses_win self.acc_num = acc_num self.drawleadstr() self.ui.exec_locked(self.window.noutrefresh) # Update the child ThreadFrames for child in self.children: child.update(curses_win, self.location, 0) self.location += 1 def get_new_tframe(self): """Create a new ThreadFrame and append it to self.children. :returns: The new ThreadFrame""" tf = CursesThreadFrame(self.ui, self.window, self.location, 0) self.location += 1 self.children.append(tf) return tf def sleeping(self, sleepsecs, remainingsecs): """Show how long we are going to sleep and sleep. :returns: Boolean, whether we want to abort the sleep""" self.drawleadstr(remainingsecs) self.ui.exec_locked(self.window.refresh) time.sleep(sleepsecs) return self.account.get_abort_event() def syncnow(self): """Request that we stop sleeping asap and continue to sync.""" # if this belongs to an Account (and not *Control), set the # skipsleep pref if isinstance(self.account, offlineimap.accounts.Account): self.ui.info("Requested synchronization for acc: %s"% self.account) self.account.config.set('Account %s'% self.account.name, 'skipsleep', '1') class CursesThreadFrame: """curses_color: current color pair for logging.""" def __init__(self, ui, acc_win, x, y): """ :param ui: is a Blinkenlights() instance :param acc_win: curses Account window""" self.ui = ui self.window = acc_win self.x = x self.y = y self.curses_color = curses.color_pair(0) #default color def setcolor(self, color, modifier=0): """Draw the thread symbol '@' in the specified color :param modifier: Curses modified, such as curses.A_BOLD """ self.curses_color = modifier | self.ui.curses_colorpair(color) self.colorname = color self.display() def display(self): def locked_display(): try: self.window.addch(self.y, self.x, '@', self.curses_color) except curses.error: # Occurs when the terminal is very small pass self.window.refresh() # lock the curses IO while fudging stuff self.ui.exec_locked(locked_display) def update(self, acc_win, x, y): """Update the xy position of the '.' (and possibly the aframe).""" self.window = acc_win self.y = y self.x = x self.display() def std_color(self): self.setcolor('black') class InputHandler(ExitNotifyThread): """Listens for input via the curses interfaces""" #TODO, we need to use the ugly exitnotifythread (rather than simply #threading.Thread here, so exiting this thread via the callback #handler, kills off all parents too. Otherwise, they would simply #continue. def __init__(self, ui): super(InputHandler, self).__init__() self.char_handler = None self.ui = ui self.enabled = Event() # We will only parse input if we are enabled. self.inputlock = RLock() # denotes whether we should be handling the next char. self.start() #automatically start the thread def get_next_char(self): """Return the key pressed or -1. Wait until `enabled` and loop internally every stdscr.timeout() msecs, releasing the inputlock. :returns: char or None if disabled while in here""" self.enabled.wait() while self.enabled.is_set(): with self.inputlock: char = self.ui.stdscr.getch() if char != -1: yield char def run(self): while True: char_gen = self.get_next_char() for char in char_gen: self.char_handler(char) #curses.ungetch(char) def set_char_hdlr(self, callback): """Sets a character callback handler. If a key is pressed it will be passed to this handler. Keys include the curses.KEY_RESIZE key. callback is a function taking a single arg -- the char pressed. If callback is None, input will be ignored.""" with self.inputlock: self.char_handler = callback # start or stop the parsing of things if callback is None: self.enabled.clear() else: self.enabled.set() def input_acquire(self): """Call this method when you want exclusive input control. Make sure to call input_release afterwards! While this lockis held, input can go to e.g. the getpass input.""" self.enabled.clear() self.inputlock.acquire() def input_release(self): """Call this method when you are done getting input.""" self.inputlock.release() self.enabled.set() class CursesLogHandler(logging.StreamHandler): """self.ui has been set to the UI class before anything is invoked""" def emit(self, record): log_str = logging.StreamHandler.format(self, record) color = self.ui.gettf().curses_color # We must acquire both locks. Otherwise, deadlock can result. # This can happen if one thread calls _msg (locking curses, then # tf) and another tries to set the color (locking tf, then curses) # # By locking both up-front here, in this order, we prevent deadlock. self.ui.tframe_lock.acquire() self.ui.lock() try: y,x = self.ui.logwin.getyx() if y or x: self.ui.logwin.addch(10) # no \n before 1st item self.ui.logwin.addstr(log_str, color) self.ui.logwin.noutrefresh() self.ui.stdscr.refresh() finally: self.ui.unlock() self.ui.tframe_lock.release() class Blinkenlights(UIBase, CursesUtil): """Curses-cased fancy UI. Notable instance variables self. ....: - stdscr: THe curses std screen - bannerwin: The top line banner window - width|height: The total curses screen dimensions - logheight: Available height for the logging part - log_con_handler: The CursesLogHandler() - threadframes: - accframes[account]: 'Accountframe'""" def __init__(self, *args, **kwargs): super(Blinkenlights, self).__init__(*args, **kwargs) CursesUtil.__init__(self) ################################################## UTILS def setup_consolehandler(self): """Backend specific console handler. Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" # create console handler with a higher log level ch = CursesLogHandler() #ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") ch.setFormatter(self.formatter) # add the handlers to the logger self.logger.addHandler(ch) # the handler is not usable yet. We still need all the # intialization stuff currently done in init_banner. Move here? return ch def isusable(s): """Returns true if the backend is usable ie Curses works.""" # Not a terminal? Can't use curses. if not sys.stdout.isatty() and sys.stdin.isatty(): return False # No TERM specified? Can't use curses. if not os.environ.get('TERM', None): return False # Test if ncurses actually starts up fine. Only do so for # python>=2.6.6 as calling initscr() twice messing things up. # see http://bugs.python.org/issue7567 in python 2.6 to 2.6.5 if sys.version_info[0:3] < (2,6) or sys.version_info[0:3] >= (2,6,6): try: curses.initscr() curses.endwin() except: return False return True def init_banner(self): self.availablethreadframes = {} self.threadframes = {} self.accframes = {} self.aflock = Lock() self.stdscr = curses.initscr() # turn off automatic echoing of keys to the screen curses.noecho() # react to keys instantly, without Enter key curses.cbreak() # return special key values, eg curses.KEY_LEFT self.stdscr.keypad(1) # wait 1s for input, so we don't block the InputHandler infinitely self.stdscr.timeout(1000) curses.start_color() # turn off cursor and save original state self.oldcursor = None try: self.oldcursor = curses.curs_set(0) except: pass self.stdscr.clear() self.stdscr.refresh() self.init_colorpairs() # set log handlers ui to ourself self._log_con_handler.ui = self self.setupwindows() # Settup keyboard handler self.inputhandler = InputHandler(self) self.inputhandler.set_char_hdlr(self.on_keypressed) self.gettf().setcolor('red') self.info(offlineimap.banner) def acct(self, *args): """Output that we start syncing an account (and start counting).""" self.gettf().setcolor('purple') super(Blinkenlights, self).acct(*args) def connecting(self, *args): self.gettf().setcolor('white') super(Blinkenlights, self).connecting(*args) def syncfolders(self, *args): self.gettf().setcolor('blue') super(Blinkenlights, self).syncfolders(*args) def syncingfolder(self, *args): self.gettf().setcolor('cyan') super(Blinkenlights, self).syncingfolder(*args) def skippingfolder(self, *args): self.gettf().setcolor('cyan') super(Blinkenlights, self).skippingfolder(*args) def loadmessagelist(self, *args): self.gettf().setcolor('green') super(Blinkenlights, self).loadmessagelist(*args) def syncingmessages(self, *args): self.gettf().setcolor('blue') super(Blinkenlights, self).syncingmessages(*args) def ignorecopyingmessage(self, *args): self.gettf().setcolor('red') super(Blinkenlights, self).ignorecopyingmessage(*args) def copyingmessage(self, *args): self.gettf().setcolor('orange') super(Blinkenlights, self).copyingmessage(*args) def deletingmessages(self, *args): self.gettf().setcolor('red') super(Blinkenlights, self).deletingmessages(*args) def addingflags(self, *args): self.gettf().setcolor('blue') super(Blinkenlights, self).addingflags(*args) def deletingflags(self, *args): self.gettf().setcolor('blue') super(Blinkenlights, self).deletingflags(*args) def callhook(self, *args): self.gettf().setcolor('white') super(Blinkenlights, self).callhook(*args) ############ Generic logging functions ############################# def warn(self, msg, minor=0): self.gettf().setcolor('red', curses.A_BOLD) super(Blinkenlights, self).warn(msg) def threadExited(self, thread): acc = self.getthreadaccount(thread) with self.tframe_lock: if thread in self.threadframes[acc]: tf = self.threadframes[acc][thread] tf.setcolor('black') self.availablethreadframes[acc].append(tf) del self.threadframes[acc][thread] super(Blinkenlights, self).threadExited(thread) def gettf(self): """Return the ThreadFrame() of the current thread.""" cur_thread = currentThread() acc = self.getthreadaccount() #Account() or None with self.tframe_lock: # Ideally we already have self.threadframes[accountname][thread] try: if cur_thread in self.threadframes[acc]: return self.threadframes[acc][cur_thread] except KeyError: # Ensure threadframes already has an account dict self.threadframes[acc] = {} self.availablethreadframes[acc] = deque() # If available, return a ThreadFrame() if len(self.availablethreadframes[acc]): tf = self.availablethreadframes[acc].popleft() tf.std_color() else: tf = self.getaccountframe(acc).get_new_tframe() self.threadframes[acc][cur_thread] = tf return tf def on_keypressed(self, key): # received special KEY_RESIZE, resize terminal if key == curses.KEY_RESIZE: self.resizeterm() return if key < 1 or key > 255: return if chr(key) == 'q': # Request to quit completely. self.warn("Requested shutdown via 'q'") offlineimap.accounts.Account.set_abort_event(self.config, 3) try: index = int(chr(key)) except ValueError: return # Key not a valid number: exit. if index >= len(self.hotkeys): # Not in our list of valid hotkeys. return # Trying to end sleep somewhere. self.getaccountframe(self.hotkeys[index]).syncnow() def sleep(self, sleepsecs, account): self.gettf().setcolor('red') self.info("Next sync in %d:%02d"% (sleepsecs / 60, sleepsecs % 60)) return super(Blinkenlights, self).sleep(sleepsecs, account) def sleeping(self, sleepsecs, remainingsecs): if not sleepsecs: # reset color to default if we are done sleeping. self.gettf().setcolor('white') accframe = self.getaccountframe(self.getthreadaccount()) return accframe.sleeping(sleepsecs, remainingsecs) def resizeterm(self): """Resize the current windows.""" self.exec_locked(self.setupwindows, True) def mainException(self): UIBase.mainException(self) def getpass(self, username, config, errmsg=None): # disable the hotkeys inputhandler self.inputhandler.input_acquire() # See comment on _msg for info on why both locks are obtained. self.lock() try: #s.gettf().setcolor('white') self.warn(" *** Input Required") self.warn(" *** Please enter password for user '%s': " % \ username) self.logwin.refresh() password = self.logwin.getstr() finally: self.unlock() self.inputhandler.input_release() return password def setupwindows(self, resize=False): """Setup and draw bannerwin and logwin. If `resize`, don't create new windows, just adapt size. This function should be invoked with CursesUtils.locked().""" self.height, self.width = self.stdscr.getmaxyx() self.logheight = self.height - len(self.accframes) - 1 if resize: if curses.is_term_resized(self.height, self.width): curses.resizeterm(self.height, self.width) self.bannerwin.resize(1, self.width) self.logwin.resize(self.logheight, self.width) self.stdscr.clear() self.stdscr.noutrefresh() else: self.bannerwin = curses.newwin(1, self.width, 0, 0) self.logwin = curses.newwin(self.logheight, self.width, 1, 0) self.draw_bannerwin() self.logwin.idlok(True) # needed for scrollok below self.logwin.scrollok(True) # scroll window when too many lines added self.draw_logwin() self.accounts = reversed(sorted(self.accframes.keys())) pos = self.height - 1 index = 0 self.hotkeys = [] for account in self.accounts: acc_win = curses.newwin(1, self.width, pos, 0) self.accframes[account].setwindow(acc_win, index) self.hotkeys.append(account) index += 1 pos -= 1 curses.doupdate() def draw_bannerwin(self): """Draw the top-line banner line.""" if curses.has_colors(): color = curses.A_BOLD | self.curses_colorpair('banner') else: color = curses.A_REVERSE self.bannerwin.clear() # Delete old content (eg before resizes) self.bannerwin.bkgd(' ', color) # Fill background with that color string = "%s %s" % (offlineimap.__productname__, offlineimap.__version__) spaces = " " * max(1, (self.width - len(offlineimap.__copyright__) - len(string) - 1)) string = "%s%s%s" % (string, spaces, offlineimap.__copyright__) self.bannerwin.addnstr(0, 0, string, self.width - 1, color) self.bannerwin.noutrefresh() def draw_logwin(self): """(Re)draw the current logwindow.""" if curses.has_colors(): color = curses.color_pair(0) #default colors else: color = curses.A_NORMAL self.logwin.move(0, 0) self.logwin.clear() self.logwin.bkgd(' ', color) self.logwin.noutrefresh() def getaccountframe(self, acc_name): """Return an AccountFrame() corresponding to acc_name. Note that the *control thread uses acc_name `None`.""" with self.aflock: # 1) Return existing or 2) create a new CursesAccountFrame. if acc_name in self.accframes: return self.accframes[acc_name] self.accframes[acc_name] = CursesAccountFrame(self, acc_name) # update the window layout self.setupwindows(resize= True) return self.accframes[acc_name] def terminate(self, *args, **kwargs): curses.nocbreak(); self.stdscr.keypad(0); curses.echo() curses.endwin() # need to remove the Curses console handler now and replace with # basic one, so exceptions and stuff are properly displayed self.logger.removeHandler(self._log_con_handler) UIBase.setup_consolehandler(self) # reset the warning method, we do not have curses anymore self.warn = super(Blinkenlights, self).warn # finally call parent terminate which prints out exceptions etc super(Blinkenlights, self).terminate(*args, **kwargs) def threadException(self, thread): #self._log_con_handler.stop() UIBase.threadException(self, thread) ================================================ FILE: offlineimap/ui/Machine.py ================================================ # Copyright (C) 2007-2018 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA try: from urllib import urlencode except ImportError: # python3 from urllib.parse import urlencode import sys import time import logging from threading import currentThread import offlineimap from offlineimap.ui.UIBase import UIBase protocol = '7.2.0' class MachineLogFormatter(logging.Formatter): """urlencodes any outputted line, to avoid multi-line output""" def format(s, record): # Mapping of log levels to historic tag names severity_map = { 'info': 'msg', 'warning': 'warn', } line = super(MachineLogFormatter, s).format(record) severity = record.levelname.lower() if severity in severity_map: severity = severity_map[severity] if hasattr(record, "machineui"): command = record.machineui["command"] whoami = record.machineui["id"] else: command = "" whoami = currentThread().getName() prefix = "%s:%s"% (command, urlencode([('', whoami)])[1:]) return "%s:%s:%s"% (severity, prefix, urlencode([('', line)])[1:]) class MachineUI(UIBase): def __init__(s, config, loglevel=logging.INFO): super(MachineUI, s).__init__(config, loglevel) s._log_con_handler.createLock() """lock needed to block on password input""" # Set up the formatter that urlencodes the strings... s._log_con_handler.setFormatter(MachineLogFormatter()) # Arguments: # - handler: must be method from s.logger that reflects # the severity of the passed message # - command: command that produced this message # - msg: the message itself def _printData(s, handler, command, msg): handler(msg, extra = { 'machineui': { 'command': command, 'id': currentThread().getName(), } }) def _msg(s, msg): s._printData(s.logger.info, '_display', msg) def warn(s, msg, minor=0): # TODO, remove and cleanup the unused minor stuff s._printData(s.logger.warning, '', msg) def registerthread(s, account): super(MachineUI, s).registerthread(account) s._printData(s.logger.info, 'registerthread', account) def unregisterthread(s, thread): UIBase.unregisterthread(s, thread) s._printData(s.logger.info, 'unregisterthread', thread.getName()) def debugging(s, debugtype): s._printData(s.logger.debug, 'debugging', debugtype) def acct(s, accountname): s._printData(s.logger.info, 'acct', accountname) def acctdone(s, accountname): s._printData(s.logger.info, 'acctdone', accountname) def validityproblem(s, folder): s._printData(s.logger.warning, 'validityproblem', "%s\n%s\n%s\n%s"% (folder.getname(), folder.getrepository().getname(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def connecting(s, reposname, hostname, port): s._printData(s.logger.info, 'connecting', "%s\n%s\n%s"% (hostname, str(port), reposname)) def syncfolders(s, srcrepos, destrepos): s._printData(s.logger.info, 'syncfolders', "%s\n%s"% (s.getnicename(srcrepos), s.getnicename(destrepos))) def syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder): s._printData(s.logger.info, 'syncingfolder', "%s\n%s\n%s\n%s\n"% (s.getnicename(srcrepos), srcfolder.getname(), s.getnicename(destrepos), destfolder.getname())) def loadmessagelist(s, repos, folder): s._printData(s.logger.info, 'loadmessagelist', "%s\n%s"% (s.getnicename(repos), folder.getvisiblename())) def messagelistloaded(s, repos, folder, count): s._printData(s.logger.info, 'messagelistloaded', "%s\n%s\n%d"% (s.getnicename(repos), folder.getname(), count)) def syncingmessages(s, sr, sf, dr, df): s._printData(s.logger.info, 'syncingmessages', "%s\n%s\n%s\n%s\n"% (s.getnicename(sr), sf.getname(), s.getnicename(dr), df.getname())) def ignorecopyingmessage(s, uid, srcfolder, destfolder): s._printData(s.logger.info, 'ignorecopyingmessage', "%d\n%s\n%s\n%s[%s]"% (uid, s.getnicename(srcfolder), srcfolder.getname(), s.getnicename(destfolder), destfolder)) def copyingmessage(s, uid, num, num_to_copy, srcfolder, destfolder): s._printData(s.logger.info, 'copyingmessage', "%d\n%s\n%s\n%s[%s]"% (uid, s.getnicename(srcfolder), srcfolder.getname(), s.getnicename(destfolder), destfolder)) def folderlist(s, list): return ("\f".join(["%s\t%s"% (s.getnicename(x), x.getname()) for x in list])) def uidlist(s, list): return ("\f".join([str(u) for u in list])) def deletingmessages(s, uidlist, destlist): ds = s.folderlist(destlist) s._printData(s.logger.info, 'deletingmessages', "%s\n%s"% (s.uidlist(uidlist), ds)) def addingflags(s, uidlist, flags, dest): s._printData(s.logger.info, "addingflags", "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), dest)) def deletingflags(s, uidlist, flags, dest): s._printData(s.logger.info, 'deletingflags', "%s\n%s\n%s"% (s.uidlist(uidlist), "\f".join(flags), dest)) def threadException(s, thread): s._printData(s.logger.warning, 'threadException', "%s\n%s"% (thread.getName(), s.getThreadExceptionString(thread))) s.delThreadDebugLog(thread) s.terminate(100) def terminate(s, exitstatus=0, errortitle='', errormsg=''): s._printData(s.logger.info, 'terminate', "%d\n%s\n%s"% (exitstatus, errortitle, errormsg)) sys.exit(exitstatus) def mainException(s): s._printData(s.logger.warning, 'mainException', s.getMainExceptionString()) def threadExited(s, thread): s._printData(s.logger.info, 'threadExited', thread.getName()) UIBase.threadExited(s, thread) def sleeping(s, sleepsecs, remainingsecs): s._printData(s.logger.info, 'sleeping', "%d\n%d"% (sleepsecs, remainingsecs)) if sleepsecs > 0: time.sleep(sleepsecs) return 0 def getpass(s, username, config, errmsg=None): if errmsg: s._printData(s.logger.warning, 'getpasserror', "%s\n%s"% (username, errmsg), False) s._log_con_handler.acquire() # lock the console output try: s._printData(s.logger.info, 'getpass', username) return (sys.stdin.readline()[:-1]) finally: s._log_con_handler.release() def init_banner(s): s._printData(s.logger.info, 'protocol', protocol) s._printData(s.logger.info, 'initbanner', offlineimap.banner) def callhook(s, msg): s._printData(s.logger.info, 'callhook', msg) ================================================ FILE: offlineimap/ui/Noninteractive.py ================================================ # Noninteractive UI # Copyright (C) 2002-2016 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging import offlineimap from offlineimap.ui.UIBase import UIBase class Basic(UIBase): """'Basic' simply sets log level to INFO.""" def __init__(self, config, loglevel = logging.INFO): return super(Basic, self).__init__(config, loglevel) class Quiet(UIBase): """'Quiet' simply sets log level to WARNING""" def __init__(self, config, loglevel = logging.WARNING): return super(Quiet, self).__init__(config, loglevel) class Syslog(UIBase): """'Syslog' sets log level to INFO and outputs to syslog instead of stdout""" def __init__(self, config, loglevel = logging.INFO): return super(Syslog, self).__init__(config, loglevel) def setup_consolehandler(self): # create syslog handler ch = logging.handlers.SysLogHandler('/dev/log') # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") ch.setFormatter(self.formatter) # add the handlers to the logger self.logger.addHandler(ch) self.logger.info(offlineimap.banner) return ch def setup_sysloghandler(self): pass # Do not honor -s (log to syslog) CLI option. ================================================ FILE: offlineimap/ui/TTY.py ================================================ # TTY UI # Copyright (C) 2002-2018 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging import sys import time from getpass import getpass from offlineimap import banner from offlineimap.ui.UIBase import UIBase class TTYFormatter(logging.Formatter): """Specific Formatter that adds thread information to the log output.""" def __init__(self, *args, **kwargs): #super() doesn't work in py2.6 as 'logging' uses old-style class logging.Formatter.__init__(self, *args, **kwargs) self._last_log_thread = None def format(self, record): """Override format to add thread information.""" #super() doesn't work in py2.6 as 'logging' uses old-style class log_str = logging.Formatter.format(self, record) # If msg comes from a different thread than our last, prepend # thread info. Most look like 'Account sync foo' or 'Folder # sync foo'. t_name = record.threadName if t_name == 'MainThread': return log_str # main thread doesn't get things prepended if t_name != self._last_log_thread: self._last_log_thread = t_name log_str = "%s:\n %s" % (t_name, log_str) else: log_str = " %s"% log_str return log_str class TTYUI(UIBase): def setup_consolehandler(self): """Backend specific console handler Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" # create console handler with a higher log level ch = logging.StreamHandler() #ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers self.formatter = TTYFormatter("%(message)s") ch.setFormatter(self.formatter) # add the handlers to the logger self.logger.addHandler(ch) self.logger.info(banner) # init lock for console output ch.createLock() return ch def isusable(self): """TTYUI is reported as usable when invoked on a terminal.""" return sys.stdout.isatty() and sys.stdin.isatty() def getpass(self, username, config, errmsg=None): """TTYUI backend is capable of querying the password.""" if errmsg: self.warn("%s: %s"% (username, errmsg)) self._log_con_handler.acquire() # lock the console output try: return getpass("Enter password for user '%s': " % username) finally: self._log_con_handler.release() def mainException(self): if isinstance(sys.exc_info()[1], KeyboardInterrupt): self.logger.warn("Timer interrupted at user request; program " "terminating.\n") self.terminate() else: UIBase.mainException(self) def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. Display a message on the screen if we pass a full minute. This implementation in UIBase does not support this, but some implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately.""" if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.info("Next refresh in %.1f minutes" % ( remainingsecs/60.0)) time.sleep(sleepsecs) return 0 ================================================ FILE: offlineimap/ui/UIBase.py ================================================ # UI base class # Copyright (C) 2002-2018 John Goerzen & contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import logging import logging.handlers import re import time import sys import traceback import threading try: from Queue import Queue except ImportError: # python3 from queue import Queue from collections import deque import offlineimap from offlineimap.error import OfflineImapError debugtypes = {'':'Other offlineimap related sync messages', 'imap': 'IMAP protocol debugging', 'maildir': 'Maildir repository debugging', 'thread': 'Threading debugging'} globalui = None def setglobalui(newui): """Set the global ui object to be used for logging.""" global globalui globalui = newui def getglobalui(): """Return the current ui object.""" global globalui return globalui class UIBase(object): def __init__(self, config, loglevel=logging.INFO): self.config = config # Is this a 'dryrun'? self.dryrun = config.getdefaultboolean('general', 'dry-run', False) self.debuglist = [] # list of debugtypes we are supposed to log self.debugmessages = {} # debugmessages in a deque(v) per thread(k) self.debugmsglen = 15 self.threadaccounts = {} # dict linking active threads (k) to account names (v) self.acct_startimes = {} # linking active accounts with the time.time() when sync started self.logfile = None self.exc_queue = Queue() # saves all occuring exceptions, so we can output them at the end self.uidval_problem = False # at least one folder skipped due to UID validity problem # create logger with 'OfflineImap' app self.logger = logging.getLogger('OfflineImap') self.logger.setLevel(loglevel) self._log_con_handler = self.setup_consolehandler() """The console handler (we need access to be able to lock it).""" ################################################## UTILS def setup_consolehandler(self): """Backend specific console handler. Sets up things and adds them to self.logger. :returns: The logging.Handler() for console output""" # create console handler with a higher log level ch = logging.StreamHandler(sys.stdout) #ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") ch.setFormatter(self.formatter) # add the handlers to the logger self.logger.addHandler(ch) self.logger.info(offlineimap.banner) return ch def setup_sysloghandler(self): """Backend specific syslog handler.""" # create syslog handler ch = logging.handlers.SysLogHandler('/dev/log') # create formatter and add it to the handlers self.formatter = logging.Formatter("%(message)s") ch.setFormatter(self.formatter) # add the handlers to the logger self.logger.addHandler(ch) def setlogfile(self, logfile): """Create file handler which logs to file.""" fh = logging.FileHandler(logfile, 'at') #fh.setLevel(logging.DEBUG) file_formatter = logging.Formatter("%(asctime)s %(levelname)s: " "%(message)s", '%Y-%m-%d %H:%M:%S') fh.setFormatter(file_formatter) self.logger.addHandler(fh) # write out more verbose initial info blurb on the log file p_ver = ".".join([str(x) for x in sys.version_info[0:3]]) msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\ "Args: %s"% (offlineimap.__version__, p_ver, sys.platform, " ".join(sys.argv)) record = logging.LogRecord('OfflineImap', logging.INFO, __file__, None, msg, None, None) fh.emit(record) def _msg(self, msg): """Display a message.""" # TODO: legacy function, rip out. self.info(msg) def info(self, msg): """Display a message.""" self.logger.info(msg) def warn(self, msg, minor=0): self.logger.warning(msg) def error(self, exc, exc_traceback=None, msg=None): """Log a message at severity level ERROR. Log Exception 'exc' to error log, possibly prepended by a preceding error "msg", detailing at what point the error occurred. In debug mode, we also output the full traceback that occurred if one has been passed in via sys.info()[2]. Also save the Exception to a stack that can be output at the end of the sync run when offlineiamp exits. It is recommended to always pass in exceptions if possible, so we can give the user the best debugging info. We are always pushing tracebacks to the exception queue to make them to be output at the end of the run to allow users pass sensible diagnostics to the developers or to solve problems by themselves. One example of such a call might be: ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in " "repo %s") """ if msg: self.logger.error("ERROR: %s\n %s"% (msg, exc)) else: self.logger.error("ERROR: %s"% (exc)) instant_traceback = exc_traceback if not self.debuglist: # only output tracebacks in debug mode instant_traceback = None # push exc on the queue for later output self.exc_queue.put((msg, exc, exc_traceback)) if instant_traceback: self.logger.error(traceback.format_tb(instant_traceback)) def registerthread(self, account): """Register current thread as being associated with an account name.""" cur_thread = threading.currentThread() if cur_thread in self.threadaccounts: # was already associated with an old account, update info self.debug('thread', "Register thread '%s' (previously '%s', now " "'%s')"% (cur_thread.getName(), self.getthreadaccount(cur_thread), account)) else: self.debug('thread', "Register new thread '%s' (account '%s')"% (cur_thread.getName(), account)) self.threadaccounts[cur_thread] = account def unregisterthread(self, thr): """Unregister a thread as being associated with an account name.""" if thr in self.threadaccounts: del self.threadaccounts[thr] self.debug('thread', "Unregister thread '%s'"% thr.getName()) def getthreadaccount(self, thr=None): """Get Account() for a thread (current if None) If no account has been registered with this thread, return 'None'.""" if thr == None: thr = threading.currentThread() if thr in self.threadaccounts: return self.threadaccounts[thr] return None def debug(self, debugtype, msg): cur_thread = threading.currentThread() if not cur_thread in self.debugmessages: # deque(..., self.debugmsglen) would be handy but was # introduced in p2.6 only, so we'll need to work around and # shorten our debugmsg list manually :-( self.debugmessages[cur_thread] = deque() self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg)) # Shorten queue if needed if len(self.debugmessages[cur_thread]) > self.debugmsglen: self.debugmessages[cur_thread].popleft() if debugtype in self.debuglist: # log if we are supposed to do so self.logger.debug("[%s]: %s" % (debugtype, msg)) def add_debug(self, debugtype): global debugtypes if debugtype in debugtypes: if not debugtype in self.debuglist: self.debuglist.append(debugtype) self.debugging(debugtype) else: self.invaliddebug(debugtype) def debugging(self, debugtype): global debugtypes self.logger.debug("Now debugging for %s: %s" % (debugtype, debugtypes[debugtype])) def invaliddebug(self, debugtype): self.warn("Invalid debug type: %s" % debugtype) def getnicename(self, object): """Return the type of a repository or Folder as string. (IMAP, Gmail, Maildir, etc...)""" prelimname = object.__class__.__name__.split('.')[-1] # Strip off extra stuff. return re.sub('(Folder|Repository)', '', prelimname) def isusable(self): """Returns true if this UI object is usable in the current environment. For instance, an X GUI would return true if it's being run in X with a valid DISPLAY setting, and false otherwise.""" return True ################################################## INPUT def getpass(self, username, config, errmsg = None): raise NotImplementedError("Prompting for a password is not supported" " in this UI backend.") def folderlist(self, folder_list): return ', '.join(["%s[%s]"% \ (self.getnicename(x), x.getname()) for x in folder_list]) ################################################## WARNINGS def msgtoreadonly(self, destfolder, uid, content, flags): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to synchronize message %d to folder %s[%s], " "but that folder is read-only. The message will not be " "copied to that folder."% ( uid, self.getnicename(destfolder), destfolder)) def flagstoreadonly(self, destfolder, uidlist, flags): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to modify flags for messages %s in folder %s[%s], " "but that folder is read-only. No flags have been modified " "for that message."% ( str(uidlist), self.getnicename(destfolder), destfolder)) def labelstoreadonly(self, destfolder, uidlist, labels): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to modify labels for messages %s in folder %s[%s], " "but that folder is read-only. No labels have been modified " "for that message."% ( str(uidlist), self.getnicename(destfolder), destfolder)) def deletereadonly(self, destfolder, uidlist): if self.config.has_option('general', 'ignore-readonly') and \ self.config.getboolean('general', 'ignore-readonly'): return self.warn("Attempted to delete messages %s in folder %s[%s], but that " "folder is read-only. No messages have been deleted in that " "folder."% (str(uidlist), self.getnicename(destfolder), destfolder)) ################################################## MESSAGES def init_banner(self): """Called when the UI starts. Must be called before any other UI call except isusable(). Displays the copyright banner. This is where the UI should do its setup -- TK, for instance, would create the application window here.""" pass def connecting(self, reposname, hostname, port): """Log 'Establishing connection to'.""" if not self.logger.isEnabledFor(logging.INFO): return displaystr = '' hostname = hostname if hostname else '' port = "%s"% port if port else '' if hostname: displaystr = ' to %s:%s' % (hostname, port) self.logger.info("Establishing connection%s (%s)"% (displaystr, reposname)) def acct(self, account): """Output that we start syncing an account (and start counting).""" self.acct_startimes[account] = time.time() self.logger.info("*** Processing account %s"% account) def acctdone(self, account): """Output that we finished syncing an account (in which time).""" sec = time.time() - self.acct_startimes[account] del self.acct_startimes[account] self.logger.info("*** Finished account '%s' in %d:%02d"% (account, sec // 60, sec % 60)) def syncfolders(self, src_repo, dst_repo): """Log 'Copying folder structure...'.""" if self.logger.isEnabledFor(logging.DEBUG): self.debug('', "Copying folder structure from %s to %s" %\ (src_repo, dst_repo)) ############################## Folder syncing def makefolder(self, repo, foldername): """Called when a folder is created.""" prefix = "[DRYRUN] " if self.dryrun else "" self.info(("{0}Creating folder {1}[{2}]".format( prefix, foldername, repo))) def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder): """Called when a folder sync operation is started.""" self.logger.info("Syncing %s: %s -> %s"% (srcfolder, self.getnicename(srcrepos), self.getnicename(destrepos))) def skippingfolder(self, folder): """Called when a folder sync operation is started.""" self.logger.info("Skipping %s (not changed)" % folder) def validityproblem(self, folder): self.uidval_problem = True self.logger.warning("UID validity problem for folder %s (repo %s) " "(saved %d; got %d); skipping it. Please see FAQ " "and manual on how to handle this."% \ (folder, folder.getrepository(), folder.get_saveduidvalidity(), folder.get_uidvalidity())) def loadmessagelist(self, repos, folder): self.logger.debug("Loading message list for %s[%s]"% ( self.getnicename(repos), folder)) def messagelistloaded(self, repos, folder, count): self.logger.debug("Message list for %s[%s] loaded: %d messages" % ( self.getnicename(repos), folder, count)) ############################## Message syncing def syncingmessages(self, sr, srcfolder, dr, dstfolder): self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % ( self.getnicename(sr), srcfolder, self.getnicename(dr), dstfolder)) def ignorecopyingmessage(self, uid, src, destfolder): """Output a log line stating which message is ignored.""" self.logger.info("IGNORED: Copy message UID %s %s:%s -> %s"% ( uid, src.repository, src, destfolder.repository)) def copyingmessage(self, uid, num, num_to_copy, src, destfolder): """Output a log line stating which message we copy.""" self.logger.info("Copy message UID %s (%d/%d) %s:%s -> %s:%s"% ( uid, num, num_to_copy, src.repository, src, destfolder.repository, destfolder)) def deletingmessages(self, uidlist, destlist): ds = self.folderlist(destlist) prefix = "[DRYRUN] " if self.dryrun else "" self.info("{0}Deleting {1} messages ({2}) in {3}".format( prefix, len(uidlist), offlineimap.imaputil.uid_sequence(uidlist), ds)) def addingflags(self, uidlist, flags, dest): self.logger.info("Adding flag %s to %d messages on %s" % ( ", ".join(flags), len(uidlist), dest)) def deletingflags(self, uidlist, flags, dest): self.logger.info("Deleting flag %s from %d messages on %s" % ( ", ".join(flags), len(uidlist), dest)) def addinglabels(self, uidlist, label, dest): self.logger.info("Adding label %s to %d messages on %s" % ( label, len(uidlist), dest)) def deletinglabels(self, uidlist, label, dest): self.logger.info("Deleting label %s from %d messages on %s" % ( label, len(uidlist), dest)) def settinglabels(self, uid, num, num_to_set, labels, dest): self.logger.info("Setting labels to message %d on %s (%d of %d): %s" % ( uid, dest, num, num_to_set, ", ".join(labels))) def collectingdata(self, uidlist, source): if uidlist: self.logger.info("Collecting data from %d messages on %s"% ( len(uidlist), source)) else: self.logger.info("Collecting data from messages on %s"% source) def serverdiagnostics(self, repository, type): """Connect to repository and output useful information for debugging.""" conn = None self._msg("%s repository '%s': type '%s'" % (type, repository.name, self.getnicename(repository))) try: if hasattr(repository, 'gethost'): # IMAP self._msg("Host: %s Port: %s SSL: %s"% (repository.gethost(), repository.getport(), repository.getssl())) try: conn = repository.imapserver.acquireconnection() except OfflineImapError as e: self._msg("Failed to connect. Reason %s" % e) else: if 'ID' in conn.capabilities: self._msg("Server supports ID extension.") #TODO: Debug and make below working, it hangs Gmail #res_type, response = conn.id(( # 'name', offlineimap.__productname__, # 'version', offlineimap.__version__)) #self._msg("Server ID: %s %s" % (res_type, response[0])) self._msg("Server welcome string: %s" % str(conn.welcome)) self._msg("Server capabilities: %s\n" % str(conn.capabilities)) repository.imapserver.releaseconnection(conn) if type != 'Status': folderfilter = repository.getconf('folderfilter', None) if folderfilter: self._msg("folderfilter= %s\n" % folderfilter) folderincludes = repository.getconf('folderincludes', None) if folderincludes: self._msg("folderincludes= %s\n" % folderincludes) nametrans = repository.getconf('nametrans', None) if nametrans: self._msg("nametrans= %s\n" % nametrans) folders = repository.getfolders() foldernames = [(f.name, f.getvisiblename(), f.sync_this) for f in folders] folders = [] for name, visiblename, sync_this in foldernames: syncstr = "" if sync_this else " (disabled)" if name == visiblename: folders.append("%s%s" % (name, syncstr)) else: folders.append("%s -> %s%s" % (name, visiblename, syncstr)) self._msg("Folderlist:\n %s\n" % "\n ".join(folders)) finally: if conn: #release any existing IMAP connection repository.imapserver.close() def savemessage(self, debugtype, uid, flags, folder): """Output a log line stating that we save a msg.""" self.debug(debugtype, "Write mail '%s:%d' with flags %s"% (folder, uid, repr(flags))) ################################################## Threads def getThreadDebugLog(self, thread): if thread in self.debugmessages: message = "\nLast %d debug messages logged for %s prior to exception:\n"\ % (len(self.debugmessages[thread]), thread.getName()) message += "\n".join(self.debugmessages[thread]) else: message = "\nNo debug messages were logged for %s."% \ thread.getName() return message def delThreadDebugLog(self, thread): if thread in self.debugmessages: del self.debugmessages[thread] def getThreadExceptionString(self, thread): message = "Thread '%s' terminated with exception:\n%s"% \ (thread.getName(), thread.exit_stacktrace) message += "\n" + self.getThreadDebugLog(thread) return message def threadException(self, thread): """Called when a thread has terminated with an exception. The argument is the ExitNotifyThread that has so terminated.""" self.warn(self.getThreadExceptionString(thread)) self.delThreadDebugLog(thread) self.terminate(100) def terminate(self, exitstatus = 0, errortitle = None, errormsg = None): """Called to terminate the application.""" #print any exceptions that have occurred over the run if not self.exc_queue.empty(): self.warn("ERROR: Exceptions occurred during the run!") if exitstatus == 0: exitstatus = 1 while not self.exc_queue.empty(): msg, exc, exc_traceback = self.exc_queue.get() if msg: self.warn("ERROR: %s\n %s"% (msg, exc)) else: self.warn("ERROR: %s"% (exc)) if exc_traceback: self.warn("\nTraceback:\n%s"% "".join( traceback.format_tb(exc_traceback))) if errormsg and errortitle: self.warn('ERROR: %s\n\n%s\n'% (errortitle, errormsg)) elif errormsg: self.warn('%s\n'% errormsg) if self.uidval_problem: self.warn('At least one folder skipped due to UID validity problem') if exitstatus == 0: exitstatus = 2 sys.exit(exitstatus) def threadExited(self, thread): """Called when a thread has exited normally. Many UIs will just ignore this.""" self.delThreadDebugLog(thread) self.unregisterthread(thread) ################################################## Hooks def callhook(self, msg): if self.dryrun: self.info("[DRYRUN] {0}".format(msg)) else: self.info(msg) ################################################## Other def sleep(self, sleepsecs, account): """This function does not actually output anything, but handles the overall sleep, dealing with updates as necessary. It will, however, call sleeping() which DOES output something. :returns: 0/False if timeout expired, 1/2/True if there is a request to cancel the timer. """ abortsleep = False while sleepsecs > 0 and not abortsleep: if account.get_abort_event(): abortsleep = True else: abortsleep = self.sleeping(10, sleepsecs) sleepsecs -= 10 self.sleeping(0, 0) # Done sleeping. return abortsleep def sleeping(self, sleepsecs, remainingsecs): """Sleep for sleepsecs, display remainingsecs to go. Does nothing if sleepsecs <= 0. Display a message on the screen if we pass a full minute. This implementation in UIBase does not support this, but some implementations return 0 for successful sleep and 1 for an 'abort', ie a request to sync immediately. """ if sleepsecs > 0: if remainingsecs//60 != (remainingsecs-sleepsecs)//60: self.logger.debug("Next refresh in %.1f minutes" % ( remainingsecs/60.0)) time.sleep(sleepsecs) return 0 ================================================ FILE: offlineimap/ui/__init__.py ================================================ # UI module # Copyright (C) 2010-2011 Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from offlineimap.ui.UIBase import getglobalui, setglobalui from offlineimap.ui import TTY, Noninteractive, Machine UI_LIST = {'ttyui': TTY.TTYUI, 'basic': Noninteractive.Basic, 'quiet': Noninteractive.Quiet, 'syslog': Noninteractive.Syslog, 'machineui': Machine.MachineUI} #add Blinkenlights UI if it imports correctly (curses installed) try: from offlineimap.ui import Curses UI_LIST['blinkenlights'] = Curses.Blinkenlights except ImportError: pass ================================================ FILE: offlineimap/ui/debuglock.py ================================================ # Locking debugging code -- temporary # Copyright (C) 2003-2015 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA from threading import Lock, currentThread import traceback logfile = open("/tmp/logfile", "wt") loglock = Lock() class DebuggingLock: def __init__(self, name): self.lock = Lock() self.name = name def acquire(self, blocking = 1): self.print_tb("Acquire lock") self.lock.acquire(blocking) self.logmsg("===== %s: Thread %s acquired lock\n"% (self.name, currentThread().getName())) def release(self): self.print_tb("Release lock") self.lock.release() def logmsg(self, msg): loglock.acquire() logfile.write(msg + "\n") logfile.flush() loglock.release() def print_tb(self, msg): self.logmsg(".... %s: Thread %s attempting to %s\n"% \ (self.name, currentThread().getName(), msg) + \ "\n".join(traceback.format_list(traceback.extract_stack()))) ================================================ FILE: offlineimap/utils/__init__.py ================================================ ================================================ FILE: offlineimap/utils/const.py ================================================ # Copyright (C) 2013-2014 Eygene A. Ryabinkin and contributors # # Collection of classes that implement const-like behaviour # for various objects. import copy class ConstProxy(object): """Implements read-only access to a given object that can be attached to each instance only once.""" def __init__(self): self.__dict__['__source'] = None def __getattr__(self, name): src = self.__dict__['__source'] if src == None: raise ValueError("using non-initialized ConstProxy() object") return copy.deepcopy(getattr(src, name)) def __setattr__(self, name, value): raise AttributeError("tried to set '%s' to '%s' for constant object"% \ (name, value)) def __delattr__(self, name): raise RuntimeError("tried to delete field '%s' from constant object"% \ (name)) def set_source(self, source): """ Sets source object for this instance. """ if (self.__dict__['__source'] != None): raise ValueError("source object is already set") self.__dict__['__source'] = source ================================================ FILE: offlineimap/utils/distro.py ================================================ # Copyright 2006-2018 Eygene A. Ryabinkin & contributors. # # Module that supports distribution-specific functions. import platform import os # Each dictionary value is either string or some iterable. # # For the former we will just return the value, for an iterable # we will walk through the values and will return the first # one that corresponds to the existing file. __DEF_OS_LOCATIONS = { 'freebsd': '/usr/local/share/certs/ca-root-nss.crt', 'openbsd': '/etc/ssl/cert.pem', 'dragonfly': '/etc/ssl/cert.pem', 'darwin': [ # MacPorts, port curl-ca-bundle '/opt/local/share/curl/curl-ca-bundle.crt', # homebrew, package openssl '/usr/local/etc/openssl/cert.pem', ], 'linux-ubuntu': '/etc/ssl/certs/ca-certificates.crt', 'linux-debian': '/etc/ssl/certs/ca-certificates.crt', 'linux-gentoo': '/etc/ssl/certs/ca-certificates.crt', 'linux-fedora': '/etc/pki/tls/certs/ca-bundle.crt', 'linux-redhat': '/etc/pki/tls/certs/ca-bundle.crt', 'linux-suse': '/etc/ssl/ca-bundle.pem', 'linux-opensuse': '/etc/ssl/ca-bundle.pem', 'linux-arch': '/etc/ssl/certs/ca-certificates.crt', } def get_os_name(): """ Finds out OS name. For non-Linux system it will be just a plain OS name (like FreeBSD), for Linux it will be "linux-", where is the name of the distribution, as returned by the first component of platform.linux_distribution. Return value will be all-lowercase to avoid confusion about proper name capitalisation. """ OS = platform.system().lower() if OS.startswith('linux'): DISTRO = platform.linux_distribution()[0] if DISTRO: OS = OS + "-%s" % DISTRO.split()[0].lower() if os.path.exists('/etc/arch-release'): OS = "linux-arch" return OS def get_os_sslcertfile_searchpath(): """Returns search path for CA bundle for the current OS. We will return an iterable even if configuration has just a single value: it is easier for our callers to be sure that they can iterate over result. Returned value of None means that there is no search path at all. """ OS = get_os_name() l = None if OS in __DEF_OS_LOCATIONS: l = __DEF_OS_LOCATIONS[OS] if not hasattr(l, '__iter__'): l = (l, ) return l def get_os_sslcertfile(): """ Finds out the location for the distribution-specific CA certificate file bundle. Returns the location of the file or None if there is no known CA certificate file or all known locations correspond to non-existing filesystem objects. """ l = get_os_sslcertfile_searchpath() if l == None: return None for f in l: assert (type(f) == type("")) if os.path.exists(f) and \ (os.path.isfile(f) or os.path.islink(f)): return f return None ================================================ FILE: offlineimap/utils/stacktrace.py ================================================ # Copyright 2013 Eygene A. Ryabinkin # Functions to perform stack tracing (for multithreaded programs # as well as for single-threaded ones). import sys import threading import traceback def dump(out): """ Dumps current stack trace into I/O object 'out' """ id2name = {} for th in threading.enumerate(): id2name[th.ident] = th.name n = 0 for i, stack in sys._current_frames().items(): out.write ("\n# Thread #%d (id=%d), %s\n" % \ (n, i, id2name[i])) n = n + 1 for f, lno, name, line in traceback.extract_stack (stack): out.write ('File: "%s", line %d, in %s' % \ (f, lno, name)) if line: out.write (" %s" % (line.strip())) out.write ("\n") ================================================ FILE: offlineimap/virtual_imaplib2.py ================================================ # Copyright (C) 2016-2016 Nicolas Sebrecht & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ The virtual imaplib2 takes care to import the correct imaplib2 library. Any internal use of imaplib2 everywhere else in offlineimap must be done through this virtual_imaplib2 or we might go into troubles. """ DESC = None _SUPPORTED_RELEASE = 2 _SUPPORTED_REVISION = 57 try: # Try any imaplib2 in PYTHONPATH first. This allows both maintainers of # distributions and developers to not work with the bundled imaplib2. from imaplib2 import * import imaplib2 as imaplib if (int(imaplib.__release__) < _SUPPORTED_RELEASE or int(imaplib.__revision__) < _SUPPORTED_REVISION): raise ImportError("The provided imaplib2 version '%s' is not supported"% imaplib.__version__) DESC = "system" except (ImportError, NameError) as e: try: from offlineimap.bundled_imaplib2 import * import offlineimap.bundled_imaplib2 as imaplib DESC = "bundled" except: print("Error while trying to import system imaplib2: %s"% e) raise # Upstream won't expose those literals to avoid erasing them with "import *" in # case they exist. __version__ = imaplib.__version__ __release__ = imaplib.__release__ __revision__ = imaplib.__revision__ ================================================ FILE: offlineimap.conf ================================================ # Offlineimap sample configuration file # This file documents *all* possible options and can be quite scary. # Looking for a quick start? Take a look at offlineimap.conf.minimal. # More details can be found at http://www.offlineimap.org . ################################################## # Overview ################################################## # The default configuration file is "~/.offlineimaprc". # # Offlineimap ships with a file named "offlineimap.conf" that you should copy to # that location and then edit. # # Offlineimap also ships a file named "offlineimap.conf.minimal" that you can # also try. It's useful if you want to get started with the most basic feature # set, and you can read about other features later with "offlineimap.conf". # # If you want to be XDG-compatible, you can put your configuration file into # "$XDG_CONFIG_HOME/offlineimap/config". ################################################## # General definitions ################################################## # NOTE 1: Settings generally support python interpolation. This means # values can contain python format strings which refer to other values # in the same section, or values in a special DEFAULT section. This # allows you for example to use common settings for multiple accounts: # # [Repository Gmail1] # trashfolder: %(gmailtrashfolder)s # # [Repository Gmail2] # trashfolder: %(gmailtrashfolder)s # # [DEFAULT] # gmailtrashfolder = [Gmail]/Papierkorb # # would set the trashfolder setting for your German Gmail accounts. # NOTE 2: Above feature implies that any '%' needs to be encoded as '%%' # NOTE 3: Any variable that is subject to the environment variables # ($NAME) and tilde (~username/~) expansions will receive tilde # expansion first and only after the environment variable will be # expanded in the resulting string. This behaviour is intentional # as it coincides with typical shell expansion strategy. # NOTE 4: multiple same-named sections. # The library used to parse the configuration file has known issue when multiple # sections have the same name. In such case, only the last section is considered. # It is strongly discouraged to have multiple sections with the same name. # See https://github.com/OfflineIMAP/offlineimap/issues/143 for more details. [general] # This specifies where Offlineimap is to store its metadata. # This directory will be created if it does not already exist. # # Tilde and environment variable expansions will be performed. # #metadata = ~/.offlineimap # This option stands in the [general] section. # # This variable specifies which accounts are defined. Separate them with commas. # Account names should be alphanumeric only. You will need to specify one # section per account below. You may not use "general" for an account name. # # Always use ASCII characters only. # accounts = Test # This option stands in the [general] section. # # Offlineimap can synchronize more than one account at a time. There are two # ways to sync accounts concurrently: # # 1. By running one offlineimap instance for each account (with the -a CLI # option). This is the recommended way. In this case, keep the following option # to 1. # # 2. By allowing offlineimap to sync more than one account for an instance (not # recommended). In this case, set the maxsyncaccounts option to a value greater # than 1. # #maxsyncaccounts = 1 # This option stands in the [general] section. # # You can specify one or more user interface. Offlineimap will try the first in # the list, and if it fails, the second, and so forth. # # The pre-defined options are: # Blinkenlights -- A fancy (terminal) interface # TTYUI -- a text-based (terminal) interface # Basic -- Noninteractive interface suitable for cron'ing # Quiet -- Noninteractive interface, generates no output # except for errors. # MachineUI -- Interactive interface suitable for machine # parsing. # # See also offlineimapui(7) # # You can override this with a command-line option -u. # #ui = basic # This option stands in the [general] section. # # If you try to synchronize messages to a folder which the IMAP server # considers read-only, Offlineimap will generate a warning. If you want # to suppress these warnings, set ignore-readonly to yes. Read-only # IMAP folders allow reading but not modification, so if you try to # change messages in the local copy of such a folder, the IMAP server # will prevent Offlineimap from propagating those changes to the IMAP # server. Note that ignore-readonly is UNRELATED to the "readonly" # setting which prevents a repository from being modified at all. # #ignore-readonly = no ########## Advanced settings # This option stands in the [general] section. # # You can give a Python source filename here and all config file # python snippets will be evaluated in the context of that file. # This allows you to e.g. define helper functions in the Python # source file and call them from this config file. You can find # an example of this in the manual. # # Tilde and environment variable expansions will be performed. # #pythonfile = ~/.offlineimap.py # This option is in the [general] section. # # By default, Offlineimap will not exit due to a network error until the # operating system returns an error code. Operating systems can sometimes take # forever to notice this. Here you can activate a timeout on the socket. This # timeout applies to individual socket reads and writes, not to an overall sync # operation. You could perfectly well have a 30s timeout here and your sync # still take minutes. # # Values in the 30-120 second range are reasonable. # # The default is to have no timeout beyond the OS. Times are given in seconds. # #socktimeout = 60 # This option stands in the [general] section. # # By default, Offlineimap will use fsync() to force data out to disk at # opportune times to ensure consistency. This can, however, reduce performance. # Users where /home is on SSD (Flash) may also wish to reduce write cycles. # Therefore, you can disable Offlineimap's use of fsync(). Doing so will come # at the expense of greater risk of message duplication in the event of a system # crash or power loss. Default is true. Set it to false to disable fsync. # # SQLite honors this option since v7.0.8+. However, those SQLite improvements # are still EXPERIMENTAL. # #fsync = true ################################################## # Mailbox name recorder ################################################## [mbnames] # Offlineimap can record your mailbox names in a format you specify. # You can define the header, each mailbox item, the separator, # and the footer. Here is an example for Mutt. # If enabled is yes, all settings except incremental must be specified, even if # they are just the empty string "". # # The header, peritem, sep, and footer are all Python expressions passed # through eval, so you can (and must) use Python quoting. # # The incremental setting controls whether the file is written after each # account completes or once all synced accounts are complete. This is useful if # an account is sightly slower than the other. It allows keeping the previous # file rather than having it partially written. # This works best with "no" if in one-shot mode started by cron or systemd # timers. Default: no. # # The following hash key are available to the expansion for 'peritem': # - accountname: the name of the corresponding account; # - foldername: the name of the folder; # - localfolders: path to the local directory hosting all Maildir # folders for the account. # # Tilde and environment variable expansions will be performed # for "filename" knob. # #enabled = no #filename = ~/Mutt/muttrc.mailboxes #header = "mailboxes " #peritem = "+%(accountname)s/%(foldername)s" #sep = " " #footer = "\n" #incremental = no # This option stands in the [mbnames] section. # # You can also specify a folderfilter. It will apply to the *translated* folder # name here, and it takes TWO arguments: accountname and foldername. In all # other ways, it will behave identically to the folderfilter for accounts. # Please see the folderfilter option for more information and examples. # # This filter can be used only to further restrict mbnames to a subset of # folders that pass the account's folderfilter. # # E.g.: with mbnames_folderfilter defined like this in the python file: # # def mbnames_folderfilter(accountname, foldername): # allowed = {'myaccount': ['folderA', 'folderB']} # if accountname in allowed: # return foldername in allowed[accountname] # return False # # For correct folderfilter with Microsoft servers, please see # http://www.offlineimap.org/doc/FAQ.html#exchange-and-office365 # #folderfilter = mbnames_folderfilter # This option stands in the [mbnames] section. # # You can customize the order in which mailbox names are listed in the generated # file by specifying a sort_keyfunc, which takes a single dict argument # containing keys 'accountname' and 'foldername'. This function will be called # once for each mailbox, and should return a suitable sort key that defines this # mailbox' position in the custom ordering. # # This is useful with e.g. Mutt-sidebar, which uses the mailbox order from the # generated file when listing mailboxes in the sidebar. # # Default setting is: #sort_keyfunc = lambda d: (d['accountname'], d['foldername']) ################################################## # Accounts ################################################## # This is an account definition clause. You'll have one of these for each # account listed in the "accounts" option in [general] section (above). [Account Test] # These settings specify the two folders that you will be syncing. # You'll need to have a "Repository ..." section for each one. localrepository = LocalExample remoterepository = RemoteExample ########## Advanced settings # This option stands in the [Account Test] section. # # You can have Offlineimap continue running indefinitely, automatically syncing # your mail periodically. If you want that, specify how frequently to do that # (in minutes) here. Fractional minutes (ie, 3.25) is allowed. # # If you want more than one account concurrently synced in one instance of # offlineimap (not recommended), don't forget to increase the maxsyncaccounts # option accordingly. # # NOTE: If you run systemd it's recommended to not enable this option and use # the systemd timer instead (see the ./contrib/systemd/ directory in the # repository). # #autorefresh = 5 # This option stands in the [Account Test] section. # # OfflineImap can replace a number of full updates by quick synchronizations. # This option is ignored if maxage or startdate are used. # # It only synchronizes a folder if # # 1) a Maildir folder has changed # # or # # 2) if an IMAP folder has received new messages or had messages deleted, ie # it does not update if only IMAP flags have changed. # # Full updates need to fetch ALL flags for all messages, so this makes quite a # performance difference (especially if syncing between two IMAP servers). # # Specify 0 for never, -1 for always (works even in non-autorefresh mode) # # A positive integer to do quick updates before doing another full # synchronization (requires autorefresh). Updates are always performed after # minutes, be they quick or full. # #quick = 10 # This option stands in the [Account Test] section. # # You can specify a pre and post sync hook to execute a external command. In # this case a call to imapfilter to filter mail before the sync process starts # and a custom shell script after the sync completes. # # The pre sync script has to complete before a sync to the account will start. # #presynchook = imapfilter -c someotherconfig.lua #postsynchook = notifysync.sh # This option stands in the [Account Test] section. # # If you have a limited amount of bandwidth available you can exclude larger # messages (e.g. those with large attachments etc). If you do this it will # appear to Offlineimap that these messages do not exist at all. They will not # be copied, have flags changed etc. For this to work on an IMAP server the # server must have server side search enabled. This works with Gmail and most # imap servers (e.g. cyrus etc) # # The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB # #maxsize = 2000000 # This option stands in the [Account Test] section. # # maxage enables you to sync only recent messages. There are two ways to specify # what "recent" means: if maxage is given as an integer, then only messages from # the last maxage days will be synced. If maxage is given as a date, then only # messages later than that date will be synced. # # Messages older than the cutoff will not be synced, their flags will not be # changed, they will not be deleted, etc. For Offlineimap it will be like these # messages do not exist. This will perform an IMAP search in the case of IMAP or # Gmail and therefore requires that the server support server side searching. # # Known edge cases are described in offlineimap(1). # # maxage is allowed only when the local folder is of type Maildir. It can't be # used with startdate. # # The maxage option expects an integer (for the number of days) or a date of the # form yyyy-mm-dd. # #maxage = 3 #maxage = 2015-04-01 # This option stands in the [Account Test] section. # # Maildir file format uses colon (:) separator between uniq name and info. # Unfortunately colon is not allowed character in windows file name. If you # enable maildir-windows-compatible option, Offlineimap will be able to store # messages on windows drive, but you will probably loose compatibility with # other programs working with the maildir. # #maildir-windows-compatible = no # This option stands in the [Account Test] section. # # Specifies if we want to sync GMail labels with the local repository. # Effective only for GMail IMAP repositories. # # Non-ASCII characters in labels are bad handled or won't work at all. # #synclabels = no # This option stands in the [Account Test] section. # # Name of the header to use for label storage. Format for the header # value differs for different headers, because there are some de-facto # "standards" set by popular clients: # # - X-Label or Keywords keep values separated with spaces; for these # you, obviously, should not have label values that contain spaces; # # - X-Keywords use comma (',') as the separator. # # To be consistent with the usual To-like headers, for the rest of header # types we use comma as the separator. # # Use ASCII characters only. # #labelsheader = X-Keywords # This option stands in the [Account Test] section. # # Set of labels to be ignored. Comma-separated list. GMail-specific # labels all start with backslash ('\'). # # Use ASCII characters only. # #ignorelabels = \Inbox, \Starred, \Sent, \Draft, \Spam, \Trash, \Important # This option stands in the [Account Test] section. # # Offlineimap can strip off some headers when your messages are propagated # back to the IMAP server. This option carries the comma-separated list # of headers to trim off. Header name matching is case-sensitive. # # This knob is respected only by IMAP-based accounts. Value of labelsheader # for GMail-based accounts is automatically added to this list, you don't # need to specify it explicitly. # # Use ASCII characters only. # #filterheaders = X-Some-Weird-Header # This option stands in the [Account Test] section. # # Use proxy connection for this account. Useful to bypass the GFW in China. # To specify a proxy connection, join proxy type, host and port with colons. # Available proxy types are SOCKS5, SOCKS4, HTTP. # You also need to install PySocks through pip. # # Currently, this feature leaks DNS support. # #proxy = SOCKS5:IP:9999 # EXPERIMENTAL: This option stands in the [Account Test] section. # # IMAP defines an encoding for non-ASCII ("international") characters, and most # IMAP servers store folder names in this encoding. Note that the IMAP 4rev1 # specification (RFC 3501) allows both UTF-8 and modified UTF-7 folder names # so it is *possible* that an IMAP server already uses UTF-8 encoded folder # names. But usually Folders that are shown as, say, "Gäste" will be represented # as "G&AOQ-ste", and by default will be synchronized like this by offlineIMAP. # # This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8 and back # in order to have nicely readable UTF-8 folder names in the local copy. # # WARNING: with this option enabled: # - compatibility with any other version is NOT GUARANTEED (including newer); # - existing set-ups will probably break. # - no support is provided. # # IMPORTANT: READ THIS SECTION if you intend to enable this feature for an # EXISTING ACCOUNT that has already been synchronized! # Enabling UTF-8 encoded folder names will change many things on the local # repository of an account, so you really have to create a new local repository # and review the configuration. The least that would happen otherwise is a # duplication of all folders containing non-ASCII characters. # But also the following functionality may change, so the configuration in the # remote repository configuration has to be reviewed/updated: # - decodefoldernames # This option is replaced by utf8foldernames and must be removed # If both utf8foldernames and decodefoldernames are enabled the synchronization # for the given account is aborted before doing any changes. # - nametrans # With utf8foldernames enabled any nametrans function will operate on the # UTF-8 encoded folder names, while even with decodefoldernames enabled they # operate on the original IMAP4-UTF-7 encoded names. # - folderfilter # Folder filters still work on the untranslated names before applying a # nametrans function, but still this operates on the UTF-8 encoded names. # - folderincludes # With utf8foldernames enabled this function expects UTF-8 encoded folder # names. # - foldersort # With utf8foldernames enabled the folder names passed to the sorting routine # will be the UTF encoded names. # - idlefolders # With utf8foldernames enabled folders passed to this function are expected to # be UTF-8 encoded. # #utf8foldernames = no # TESTING: This option stands in the [Account Test] section. # # Use authproxy connection for this account. Useful to bypass the GFW in China. # Set this if you wish to use a proxy for authentication but not for IMAP. # If not explicitly set, this option defaults to use the proxy socket # (so as to be compatible with prior config files). # If that is specifically NOT desired, use authproxy = '' # # To specify a proxy connection, join proxy type, host and port with colons. # Available proxy types are SOCKS5, SOCKS4, HTTP. # You also need to install PySocks through pip or your distro package manager. # #authproxy = SOCKS5:IP:9999 [Repository LocalExample] # Each repository requires a "type" declaration. The types supported for # local repositories are Maildir, GmailMaildir and IMAP. # type = Maildir # This option stands in the [Repository LocalExample] section. # # Specify local repository. Your IMAP folders will be synchronized # to maildirs created under this path. Offlineimap will create the # maildirs for you as needed. # localfolders = ~/Test # This option stands in the [Repository LocalExample] section. # # You can specify the "folder separator character" used for your Maildir # folders. It is inserted in-between the components of the tree. If you # want your folders to be nested directories, set it to "/". 'sep' is # ignored for IMAP repositories, as it is queried automatically. # Otherwise, default value is ".". # # Don't use quotes. # #sep = . # This option stands in the [Repository LocalExample] section. # # startdate syncs mails starting from a given date. It applies the date # restriction to LocalExample only. The remote repository MUST be empty # at the first sync where this option is used. # # Unlike maxage, this is supported for IMAP-IMAP sync. # # startdate can't be used with maxage. # # The startdate option expects a date in the format yyyy-mm-dd. # #startdate = 2015-04-01 # This option stands in the [Repository LocalExample] section. # # Propagate deletions from local to remote. Messages deleted in this repository # won't get deleted on remote if set to "no". Default is yes. # # See sync_deletes in the RemoteExample section, too. # #sync_deletes = yes # This option stands in the [Repository LocalExample] section. # # Some users may not want the atime (last access time) of folders to be # modified by Offlineimap. If 'restoreatime' is set to yes, Offlineimap # will restore the atime of the "new" and "cur" folders in each maildir # folder to their original value after each sync. # # In nearly all cases, the default should be fine. # #restoreatime = no # This option stands in the [Repository LocalExample] section. # # Set modification time of messages basing on the message's "Date" header. This # option makes sense for the Maildir type, only. # # This is useful if you are doing some processing/finding on your Maildir (for # example, finding messages older than 3 months), without parsing each # file/message content. # # This option is not compatible with -q (quick mode) CLI option for GmailMaildir # types. # # Default: no. # #utime_from_header = no # This option stands in the [Repository LocalExample] section. # # This option is similar to "utime_from_header" and could be used as a # complementary feature to keep track of a message date. This option only # makes sense for the Maildir type. # # By default each message is stored in a file which prefix is the fetch # timestamp and an order rank such as "1446590057_0". In a multithreading # environment message are fetched in a random order, then you can't trust # the filename to sort your boxes. # # If set to "yes" the filename prefix is built from the message "Date" header # (which should be present) or the "Received-date" if "Date" is not # found. If neither "Received-date" nor "Date" is found, the current system # date is used. Now you can quickly sort your messages using their filenames. # # Used in combination with "utime_from_header" all your message would be in # order with the correct mtime attribute. # #filename_use_mail_timestamp = no # This option stands in the [Repository LocalExample] section. # # Map IMAP [user-defined] keywords to lowercase letters, similar to Dovecot's # format described in http://wiki2.dovecot.org/MailboxFormat/Maildir . This # option makes sense for the Maildir type, only. # # Configuration example: # customflag_x = some_keyword # # With the configuration example above enabled, all IMAP messages that have # 'some_keyword' in their FLAGS field will have an 'x' in the flags part of the # maildir filename: # 1234567890.M20046P2137.mailserver,S=4542,W=4642:2,Sx # # Valid fields are customflag_[a-z], valid values are whatever the IMAP server # allows. # # Comparison in Offlineimap is case-sensitive. # #customflag_a = some_keyword #customflag_b = $OtherKeyword #customflag_c = NonJunk #customflag_d = ToDo [Repository GmailLocalExample] # This type of repository enables syncing of Gmail. All Maildir # configuration settings are also valid here but the utime_from_header. # # This is a separate Repository type from Maildir because it involves # some extra overhead which sometimes may be significant. We look for # modified tags in local messages by looking only to the files # modified since last run. This is usually rather fast, but the first # time Offlineimap runs with synclabels enabled, it will have to check # the contents of all individual messages for labels and this may take # a while. # type = GmailMaildir [Repository RemoteExample] # The remote repository. We only support IMAP or Gmail here. # type = IMAP # This option stands in the [Repository RemoteExample] section. # # Configure which address family to use for the connection. If not specified, # AF_UNSPEC is used as a fallback (default). # # AF_INET6: #ipv6 = True # # AF_INET: #ipv6 = False # These options stands in the [Repository RemoteExample] section. # # The following can fetch the account credentials via a python expression that # is parsed from the pythonfile parameter. For example, a function called # "getcredentials" that parses a file "filename" and returns the account # details for "hostname". # # #remotehosteval = getcredentials("filename", "hostname", "hostname") # # The returned value must be int type. #remoteporteval = getcredentials("filename", "hostname", "port") # # The returned value must be unicode type. #remoteusereval = getcredentials("filename", "hostname", "user") # # The returned value must be unicode type. #remotepasseval = getcredentials("filename", "hostname", "passwd") # This option stands in the [Repository RemoteExample] section. # # Specify the remote hostname. # remotehost = examplehost # This option stands in the [Repository RemoteExample] section. # # Whether or not to use STARTTLS. STARTTLS allows to upgrade a plain connection # to TLS or SSL after negotiation with the server. While a server might pretend # to support STARTTLS, the communication might not be properly established or # the secure tunnel might be broken in some way. In this case you might want to # disable STARTTLS. Unless you hit issues with STARTTLS, you are strongly # encouraged to keep STARTTLS enabled. # # STARTTLS can be used even if the 'ssl' option is disabled. If you want to # _force_ STARTTLS, you might need to disable 'ssl'. # # Default is yes. # #starttls = yes # This option stands in the [Repository RemoteExample] section. # # Whether or not to use SSL. # # Note: be care to configure the 'remotehost' line with the domain name defined # in the certificate. E.g., if you trust your provider and want to use the # certificate it provides on a shared server. Otherwise, Offlineimap will stop # and say that the domain is not named in the certificate. # # Default is yes. # #ssl = yes # This option stands in the [Repository RemoteExample] section. # # SSL Client certificate (optional). # # Tilde and environment variable expansions will be performed. # #sslclientcert = /path/to/file.crt # This option stands in the [Repository RemoteExample] section. # # SSL Client key (optional). # # Tilde and environment variable expansions will be performed. # #sslclientkey = /path/to/file.key # This option stands in the [Repository RemoteExample] section. # # SSL CA Cert(s) to verify the server cert against (optional). # No SSL verification is done without this option. If it is # specified, the CA Cert(s) need to verify the Server cert AND # match the hostname (* wildcard allowed on the left hand side) # The certificate should be in PEM format. # # Tilde and environment variable expansions will be performed. # # Special value OS-DEFAULT makes Offlineimap to automatically # determine system-wide location of standard trusted CA roots file # for known OS distributions and use the first bundle encountered # (if any). If no system-wide CA bundle is found, Offlineimap # will refuse to continue; this is done to prevent creation # of false security expectations ("I had configured CA bundle, # thou certificate verification shalt be present"). # # You can also use fingerprint verification via cert_fingerprint. # See below for more verbose explanation. # #sslcacertfile = /path/to/cacertfile.crt # This option stands in the [Repository RemoteExample] section. # # If you connect via SSL/TLS (ssl = yes) and you have no CA certificate # specified, Offlineimap will refuse to sync as it connects to a server # with an unknown "fingerprint". If you are sure you connect to the # correct server, you can then configure the presented server # fingerprint here. Offlineimap will verify that the server fingerprint # has not changed on each connect and refuse to connect otherwise. # # You can also configure fingerprint validation in addition to # CA certificate validation above and it will check both: # Offlineimap fill verify certificate first and if things will be fine, # fingerprint will be validated. # # Multiple fingerprints can be specified, separated by commas. # # In Windows, Microsoft uses the term "thumbprint" instead of "fingerprint". # # Supported fingerprint hashes are sha512, sha384, sha256, sha224 and sha1. # Fingerprints must be in hexadecimal form without leading '0x', and may have # the separating colons. This is non case-sensitive. # Examples: # sha1 "bbfe29cf97acb204591edbafe0aa8c8f914287c9". # sha1 with colons "BB:FE:29:CF:97:AC:B2:04:59:1E:DB:AF:E0:AA:8C:8F:91:42:87:C9" # #cert_fingerprint = [, ] # This option stands in the [Repository RemoteExample] section. # # Set SSL version to use (optional). # # It is best to leave this unset, in which case the correct version will be # automatically detected. In rare cases, it may be necessary to specify a # particular version from: tls1, tls1_1, tls1_2, ssl3, ssl23. # # tls1_1 and tls1_2 are available with OpenSSL since v1.0.1. # # ssl23 automatically selects the highest protocol version that both the client # and server support. Despite the name, this option can select “TLS” protocols # as well as “SSL”. # # Be aware that a MITM attack can consist in downgrading the protocol version # which is used upon client/server agreement. So, they might fallback to the # less secure available protocol. Hence, it is considered more safe to manually # define the protocol version. # # See the configuration option tls_level to disable insecure protocols. # #ssl_version = ssl23 # This option stands in the [Repository RemoteExample] section. # # TLS support level (optional). # # Specify the level of support that should be allowed for this repository. # Can be used to enable insecure SSL versions as defined by imaplib2. # See, IETF https://tools.ietf.org/html/rfc6176 to know more. # # Supported values are: # tls_secure, tls_no_ssl, tls_compat (default). # # Current mapping: # - tls_secure: # - tls1_1 # - tls1_2 # - tls_no_ssl: # - all tls_secure # - tls1 (less desirable than tls1_1 or higher) # - tls_compat # - all tls_no_ssl # - ssl3 (less desirable than tls1) # - ssl23 (can fallback up to ssl3) # # When tls_level is not set to tls_compat and ssl is still enabled, # the ssl_version configuration option must be explicitly set. # #tls_level = tls_compat # This option stands in the [Repository RemoteExample] section. # # Specify the port. If not specified, use a default port. # #remoteport = 993 # This option stands in the [Repository RemoteExample] section. # # Specify the remote user name. # remoteuser = username # This option stands in the [Repository RemoteExample] section. # # Specify the user to be authorized as. Sometimes we want to # authenticate with our login/password, but tell the server that we # really want to be treated as some other user; perhaps server will # allow us to do that (or maybe not). Some IMAP servers migrate # account names using this functionality: your credentials remain # intact, but remote identity changes. # # Currently this variable is used only for SASL PLAIN authentication # mechanism, so consider using auth_mechanisms to prioritize PLAIN # or even make it the only mechanism to be tried. # #remote_identity = authzuser # This option stands in the [Repository RemoteExample] section. # # Specify which authentication/authorization mechanisms we should try and the # order in which Offlineimap will try them. # # NOTE: any given mechanism will be tried ONLY if it is supported by the remote # IMAP server. # # Default value is ranged is from strongest to more weak ones. Due to technical # limitations, if GSSAPI is set, it will be tried first, no matter where it was # specified in the list. # #auth_mechanisms = GSSAPI, XOAUTH2, CRAM-MD5, PLAIN, LOGIN # This option stands in the [Repository RemoteExample] section. # # XOAUTH2 authentication (for instance, to use with Gmail). # # This option was tested on Gmail only, but should work with type = IMAP for # compatible servers. # # For Gmail (and maybe others), XOAUTH2 requires ssl. This means that STARTTLS # won't work and that Offlineimap will perform certificate validation. IOW, the # following configuration is used: # - sslcacertfile: MUST BE correctly configured # - ssl = yes (optional, will be used anyway) # - starttls = no (optional, will be tried but won't work anyway) # # Mandatory parameters are "oauth2_client_id", "oauth2_client_secret" and # either "oauth2_refresh_token" or "oauth2_access_token". XOAUTH2 mechanism # won't be tried if both oauth2_refresh_token and oauth2_access_token are not # set. # # See below to learn how to get those. # # Specify the OAuth2 client id and secret to use for the connection.. # Here's how to register an OAuth2 client for Gmail, as of 2017-05-15: # - Go to the Gmail API overview console # https://console.developers.google.com/apis/api/gmail.googleapis.com/overview # - Create a new project, name doesn't matter, e.g. 'gmail-sync-bob' # - In API & Auth, select Credentials # - Once created, click 'Enable' # - Click 'Create credentials' in the enabled API overview # - In 'Add credentials to your project' select 'Gmail API' as the # API type, and 'Other UI ...' (not 'Other non-UI ...') for # 'Where will you be calling the API from?'. For 'What data will # you be accessing?' select 'User data'. # - Click 'What credentials do I need?' # - Create an arbitrary 'Create an OAuth 2.0 client ID', # e.g. 'gmail-sync-bob-client'. For 'Set up the OAuth 2.0 consent # screen' select an arbitrary 'Product name shown to users', # e.g. 'gmail-sync-bob-client' & click 'Continue'. # - This gives you your client ID displayed on the screen. Click # 'Download' to get a JSON file that also has the client secret. # #oauth2_client_id = YOUR_CLIENT_ID #oauth2_client_secret = YOUR_CLIENT_SECRET # # The return values must be bytes. #oauth2_client_id_eval = get_client_id("accountname") #oauth2_client_secret_eval = get_client_secret("accountname") # # Specify the refresh token to use for the connection to the mail server. # Here's an example of a way to get a refresh token: # - Clone this project: https://github.com/google/gmail-oauth2-tools # - Type the following command-line in a terminal and follow the instructions # python python/oauth2.py --generate_oauth2_token \ # --client_id=YOUR_CLIENT_ID --client_secret=YOUR_CLIENT_SECRET # - Access token can be obtained using refresh token with command # python python/oauth2.py --user=YOUR_EMAIL --client_id=YOUR_CLIENT_ID # --client_secret=YOUR_CLIENT_SECRET --refresh_token=REFRESH_TOKEN # # Access tokens have limited lifetimes. If you need access beyond the lifetime # of a single access token, you should use a refresh token. A refresh token # allows Offlineimap to obtain new access tokens. # # If you want to use a refresh token, make sure you disabled/removed any # oauth2_access_token option. The access token is downloaded from the URL # defined in the oauth2_request_url configuration option. # # If the type of the remote is IMAP, oauth2_request_url MUST be defined. # For Gmail, the default URL is https://accounts.google.com/o/oauth2/token. # # If you're experiencing issues, please read the "Known issues" section of the # manual. # #oauth2_access_token = ACCESS_TOKEN #oauth2_request_url = https://accounts.google.com/o/oauth2/token #oauth2_refresh_token = REFRESH_TOKEN # # The returned values must be bytes. #oauth2_access_token_eval = get_access_token("accountname") #oauth2_refresh_token_eval = get_refresh_token("accountname") ########## Passwords # There are six ways to specify the password for the IMAP server: # # 1. No password at all specified in the config file. # If a matching entry is found in ~/.netrc (see netrc (5) for # information) this password will be used. Do note that netrc only # allows one entry per hostname. If there is no ~/.netrc file but # there is an /etc/netrc file, the password will instead be taken # from there. Otherwise you will be prompted for the password when # Offlineimap starts when using a UI that supports this. # # 2. The remote password stored in this file with the remotepass # option. Save this file with the UTF-8 encoding if your server expect UTF-8 # encoded password. # # Any '%' needs to be encoded as '%%'. Example: #remotepass = myp%%ssword # Real password is myp%ssword # # 3. The remote password stored as a single line in an external file, which is # referenced by the remotefile option. Must be UTF-8 encoded. Example: #remotepassfile = ~/Password.IMAP.Account1 # # 4. With a preauth tunnel. With this method, you invoke an external # program that is guaranteed *NOT* to ask for a password, but rather # to read from stdin and write to stdout an IMAP protocol stream that # begins life in the PREAUTH state. When you use a tunnel, you do # NOT specify a user or password (if you do, they'll be ignored.) # Instead, you specify a preauthtunnel, as this example illustrates # for Courier IMAP on Debian: #preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' # # 5. If you are using Kerberos and have the Python gssapi package # installed, you should not specify a remotepass. If the user has a # valid Kerberos TGT, Offlineimap will figure out the rest all by # itself, and fall back to password authentication if needed. # # 6. Using arbitrary python code. With this method, you invoke a # function from your pythonfile. To use this method assign the name # of the function to the variable 'remotepasseval'. Example: #remotepasseval = get_password("imap.example.net") # You can also query for the username: #remoteusereval = get_username("imap.example.net") # This method can be used to design more elaborate setups, e.g. by # querying the gnome-keyring via its python bindings. ########## Advanced settings # These options stands in the [Repository RemoteExample] section. # # Tunnels. There are two types: # # - preauth: they teleport your connection to the remote system # and you don't need to authenticate yourself there; the sole # fact that you succeeded to get the tunnel running is enough. # This tunnel type was explained above in the 'Passwords' section. # # - transport: the just provide the transport (probably encrypted) # to the IMAP server, but you still need to authenticate at the # IMAP server. # # Tunnels are currently working only with IMAP servers and their # derivatives (GMail currently). Additionally, for GMail accounts # preauth tunnel settings are ignored: we don't believe that there # are ways to preauthenticate at Google mail system IMAP servers. # # You must choose at most one tunnel type, be wise M'Lord! # #preauthtunnel = ssh -q imaphost '/usr/bin/imapd ./Maildir' #transporttunnel = openssl s_client -host myimap -port 993 -quiet # This option stands in the [Repository RemoteExample] section. # # Some IMAP servers need a "reference" which often refers to the "folder root". # # This is most commonly needed with UW IMAP, where you might need to specify the # directory in which your mail is stored. The 'reference' value will be prefixed # to all folder paths referring to that repository. E.g. accessing folder 'INBOX' # with "reference = Mail" will try to access Mail/INBOX. # # The nametrans and folderfilter functions will apply to the full path, # including the reference prefix. Most users will not need this. # #reference = Mail # DEPRECATED: This option stands in the [Repository RemoteExample] section. # # IMAP defines an encoding for non-ASCII ("international") characters. Enable # this option if you want to decode them to the nowadays ubiquitous UTF-8. # # Note that the IMAP 4rev1 specification (RFC 3501) allows both UTF-8 and # modified UTF-7 folder names. # # This option converts IMAP folder names from IMAP4-UTF-7 to UTF-8. # # NOTE/LIMITATION: # - The reencoding is applied *after* a nametrans function that may be given, # so it is important to note that nametrans will work on the undecoded # UTF-7 names. # - This option only works from a remote IMAP to a local Maildir repository # - It only works *once*, so it can only be used for one-off backups # (see https://github.com/OfflineIMAP/offlineimap/issues/299 and especially # https://github.com/OfflineIMAP/offlineimap/issues/299#issuecomment-331243827) # # WARNING: with this option enabled: # - compatibility with any other version is NOT GUARANTED (including newer); # - no support is provided. # # DEPRECATION: # This option is only there for backward compatibility with existing set-ups. # For newly created accounts please use the utf8foldernames option on account # level. # # This feature was merged because it's small changes in the code. However, this # might seriously decrease the stability of the program. That's why it will # likely never be marked stable. The approach is: if it works for you, you're # lucky and you might choose to go for it. If it doesn't, sorry but this feature # is known to not work in many cases and it's not available to you. Enabling # this feature might really be a poor choice for the future since it's not # supported at all and new releases might break the setup. # #decodefoldernames = no # This option stands in the [Repository RemoteExample] section. # # In between synchronisations, Offlineimap can monitor mailboxes for new # messages using the IDLE command. If you want to enable this, specify here the # folders you wish to monitor. IMAP protocol requires a separate connection for # each folder monitored in this way, so setting this option will force settings # for: # # - maxconnections: to be at least the number of folders you give # - holdconnectionopen: to be true # - keepalive: to be 29 minutes unless you specify otherwise # - singlethreadperfolder: to be true # # The presynchook and postsynchook are executed for each new synchronisation per # folder. # # This feature isn't complete and may well have problems. See the "Known Issues" # entry in the manual for more details. # # This option should return a Python list. For example # #idlefolders = ['INBOX', 'INBOX.Alerts'] # This option stands in the [Repository RemoteExample] section. # # Offlineimap can use a compressed connection to the IMAP server. # This can result in faster downloads for some cases. # #usecompression = yes # This option stands in the [Repository RemoteExample] section. # # Offlineimap can use multiple connections to the server in order # to perform multiple synchronization actions simultaneously. # This may place a higher burden on the server. In most cases, # setting this value to 2 or 3 will speed up the sync, but in some # cases, it may slow things down. The safe answer is 1. You should # probably never set it to a value more than 5. # #maxconnections = 2 # This option stands in the [Repository RemoteExample] section. # # If you want to ensure that only one single thread is used to synchronize each # folder, set this to 'yes'. If this is set, only one thread will be used to # copy messages for each folder, but up to maxconnections threads will be used # overall, copying different folders in parallel. This option is required to # download in UIDs order. # # If this is unset (the default), then up to maxconnections threads are used # across all currently syncing folders. This option is sightly recommended in # IMAP/IMAP mode (no maildir). # #singlethreadperfolder = no # This option stands in the [Repository RemoteExample] section. # # Offlineimap normally closes IMAP server connections between refreshes if # the global option autorefresh is specified. If you wish it to keep the # connection open, set this to true. If not specified, the default is # false. Keeping the connection open means a faster sync start the # next time and may use fewer server resources on connection, but uses # more server memory. This setting has no effect if autorefresh is not set. # #holdconnectionopen = no # This option stands in the [Repository RemoteExample] section. # # If you want to have "keepalives" sent while waiting between syncs, specify the # amount of time IN SECONDS between keepalives here. Note that sometimes more # than this amount of time might pass, so don't make it tight. This setting has # no effect if autorefresh and holdconnectionopen are not both set. # #keepalive = 60 # This option stands in the [Repository RemoteExample] section. # # Normally, Offlineimap will expunge deleted messages from the server. You can # disable that if you wish. This means that Offlineimap will mark them deleted # on the server, but not actually delete them. You must use some other IMAP # client to delete them if you use this setting; otherwise, the messages will # just pile up there forever. Therefore, this setting is definitely NOT # recommended for a long term. # # Default is yes. # #expunge = no # This option stands in the [Repository RemoteExample] section. # # Specify whether to process all mail folders on the server, or only # those listed as "subscribed". # # Default is no. # #subscribedonly = no # This option stands in the [Repository RemoteExample] section. # # You can specify a folder translator. This must be a eval-able. # # Python expression that takes a foldername arg and returns the new value. A # lambda function is suggested. # # WARNING: you MUST construct it so that it NEVER returns the same value for two # folders, UNLESS the second values are filtered out by folderfilter below. # Failure to follow this rule will result in undefined behavior. # # If you enable nametrans, you will likely need to set the reversed nametrans on # the other side. See the user documentation for details and use cases. They # are also online at: http://www.offlineimap.org/doc/nametrans.html # # This example below will remove "INBOX." from the leading edge of folders # (great for Courier IMAP users). # #nametrans = lambda foldername: re.sub('^INBOX\.', '', foldername) # # Using Courier remotely and want to duplicate its mailbox naming locally? Try # this: # #nametrans = lambda foldername: re.sub('^INBOX\.*', '.', foldername) # This option stands in the [Repository RemoteExample] section. # # Determines if folderfilter will be invoked on each run (dynamic folder # filtering) or filtering status will be determined at startup (default # behaviour). # #dynamic_folderfilter = False # This option stands in the [Repository RemoteExample] section. # # You can specify which folders to sync using the folderfilter setting. You can # provide any python function (e.g. a lambda function) which will be invoked for # each foldername. If the filter function returns True, the folder will be # synced, if it returns False, it. # # The folderfilter operates on the *UNTRANSLATED* name (before any nametrans # translation takes place). # # Example 1: synchronizing only INBOX and Sent. # #folderfilter = lambda foldername: foldername in ['INBOX', 'Sent'] # # Example 2: synchronizing everything except Trash. # #folderfilter = lambda foldername: foldername not in ['Trash'] # # Example 3: Using a regular expression to exclude Trash and all folders # containing the characters "Del". # #folderfilter = lambda foldername: not re.search('(^Trash$|Del)', foldername) # # If folderfilter is not specified, ALL remote folders will be synchronized. # # You can span multiple lines by indenting the others. (Use backslashes at the # end when required by Python syntax) For instance: # #folderfilter = lambda foldername: foldername in [ # 'INBOX', 'Sent Mail', # 'Deleted Items', 'Received'] # This option stands in the [Repository RemoteExample] section. # # You can specify folderincludes to include additional folders. It should # return a Python list. This might be used to include a folder that was # excluded by your folderfilter rule, to include a folder that your server does # not specify with its LIST option, or to include a folder that is outside your # basic reference. # # The 'reference' value will not be prefixed to this folder name, even if you # have specified one. For example: # #folderincludes = ['debian.user', 'debian.personal'] # This option stands in the [Repository RemoteExample] section. # # If you do not want to have any folders created on this repository, # set the createfolders variable to False, the default is True. Using # this feature you can e.g. disable the propagation of new folders to # the new repository. # #createfolders = True # This option stands in the [Repository RemoteExample] section. # # Propagate deletions from remote to local. Messages deleted in this repository # won't get deleted on the local repository if set to "no". Default is yes. # # See sync_deletes in the LocalExample section, too. # #sync_deletes = yes # This option stands in the [Repository RemoteExample] section. # # 'foldersort' determines how folders are sorted. # # This affects order of synchronization and mbnames. The expression should # return -1, 0, or 1, as the default Python cmp() does. The two arguments, x # and y, are strings representing the names of the folders to be sorted. The # sorting is applied *AFTER* nametrans, if any. The default is to sort IMAP # folders alphabetically (case-insensitive). Usually, you should never have to # modify this. To eg. reverse the sort: # #foldersort = lambda x, y: -cmp(x, y) # This option stands in the [Repository RemoteExample] section. # # Enable 1-way synchronization. When setting 'readonly' to True, this # repository will not be modified during synchronization. Usefull to # e.g. backup an IMAP server. The readonly setting can be applied to any # type of Repository (Maildir, Imap, etc). # # Notice that this does NOT mean that the local Maildir can be used with a # read-only filesystem When OfflineImap uploads a new email, it is assigned a # new UID by the remote which is stored in the filename. Otherwise, Offlineimap # would have no way to map the UID to the file. # #readonly = False # This option stands in the [Repository RemoteExample] section. # # You can specify a newmail hook to execute an external command upon receipt # of new mail in the INBOX. # # The pythonfile must be set to import any required library. # # This example plays a sound file of your chosing when new mail arrives. # #newmail_hook = lambda: os.system( #"cvlc --play-and-stop --play-and-exit /path/to/sound/file.mp3 > /dev/null 2>&1") # This option stands in the [Repository RemoteExample] section. [TESTING] # # Operating under extreme network conditions (China) network connectivity # especially for SSL can be so bad and so slow that absolutely every possible # kind of connectivity error that can occur does occur. # # Rather than have offlineimap exit on errors it may be preferable to have it # simply retry fetching of messages dozens of times. The alternative is to # restart offlineimap in a constant loop, which may involve using significant # CPU cycles (if the repository is large) to load up the UID database again. # # This option is best utilised in combination with socktimeout, which catches # instances of interference by e.g. the GFW at the TCP layer, as well as China # ISPs simply not coping. # # Default value: retrycount = 2 # #retrycount = 2 # This option stands in the [Repository RemoteExample] section. # # If offlineimap is having troubles to download some UIDS, it's possible to get # them ignored in a list. This only ignore the download. # # The function must return the list of UIDs (integers), None otherwise. It is # passed the folder name (using the remote name with the remote separator, # likely '/'). # #copy_ignore_eval = lambda foldername: {'INBOX': [2, 3, 4]}.get(foldername) [Repository GmailExample] # A repository using Gmail's IMAP interface. # # Any configuration parameter of "IMAP" type repositories can be used here. # Only "remoteuser" (or "remoteusereval" ) is mandatory. Default values for # other parameters are OK, and you should not need fiddle with those. # # The Gmail repository provides default values for "remotehost", # "remoteport", "tunnel" and "ssl". For the defaults we use, see: # # http://mail.google.com/support/bin/answer.py?answer=78799&topic=12814 # # This means ssl is enabled and must be configured correctly when connecting to # Gmail. # # In addition we provide defaults for "oauth2_request_url", # "trashfolder" and "spamfolder". # # All of the defaults we provide can be overriden. E.g. you can # override "remotehost"/"remoteport"/"ssl" if you'd like to connect to # imap.gmail.com via a local stunnel instead of directly. # # To enable GMail labels synchronisation, set the option "synclabels" in the # corresponding "Account" section. # # Side note: Gmail will keep the deleted emails in "Gmail\All Mail" unless you # defined it differently in your online settings. # type = Gmail # This option stands in the [Repository GmailExample] section. # # Specify the Gmail user name. This is the only mandatory parameter. # remoteuser = username@gmail.com # This option stands in the [Repository GmailExample] section. # # The trash folder name may be different from [Gmail]/Trash due to localization. # You should look for the localized names of the spam folder too: "spamfolder" # tunable will help you to override the standard name. # # For example on German Gmail, this setting should be: # #trashfolder = [Gmail]/Papierkorb ================================================ FILE: offlineimap.conf.minimal ================================================ # Sample minimal config file. Copy this to ~/.offlineimaprc and edit to # get started fast. [general] accounts = Test [Account Test] localrepository = Local remoterepository = Remote [Repository Local] type = Maildir localfolders = ~/Test [Repository Remote] type = IMAP remotehost = examplehost remoteuser = jgoerzen ================================================ FILE: offlineimap.py ================================================ #!/usr/bin/env python2 # Startup from single-user installation # Copyright (C) 2002-2018 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os import sys from offlineimap import OfflineImap oi = OfflineImap() oi.run() ================================================ FILE: requirements.txt ================================================ # Requirements six gssapi[kerberos] portalocker[cygwin] rfc6555 ================================================ FILE: scripts/get-repository.sh ================================================ #!/bin/sh # # Licence: this file is in the public domain. # # Download and configure the repositories of the website or wiki. repository=$1 github_remote=$2 # # TODO # final_note () { cat < $ cd ./$1 $ git remote add myfork https://github.com//.git EOF } setup () { target_dir=$1 remote_url=$2 # Adjust $PWD if necessary. test -d scripts || cd .. if test ! -d scripts then echo "cannot figure the correct workdir..." exit 2 fi if test -d $target_dir then echo "Directory '$target_dir' already exists..." exit 3 fi git clone "${remote_url}.git" "$1" echo '' if test $? -gt 0 then echo "Cannot fork $remote_url to $1" exit 4 fi } configure_website () { renderer='./render.sh' echo "Found Github username: '$1'" echo "If it's wrong, please fix the script ./website/render.sh" cd ./website if test $? -eq 0 then sed -r -i -e "s,{{USERNAME}},$1," "$renderer" cd .. else echo "ERROR: could not enter ./website. (?)" fi } configure_wiki () { : # noop } test n$github_remote = 'n' && github_remote='origin' # Get Github username. #offlineimap_url="$(git config --local --get remote.origin.url)" offlineimap_url="$(git config --local --get remote.nicolas33.url)" username=$(echo $offlineimap_url | sed -r -e 's,.*github.com.([^/]+)/.*,\1,') case n$repository in nwebsite) upstream=https://github.com/OfflineIMAP/offlineimap.github.io setup website "$upstream" configure_website "$username" final_note website "$upstream" ;; nwiki) upstream=https://github.com/OfflineIMAP/offlineimap.wiki setup wiki "$upstream" configure_wiki final_note wiki "$upstream" ;; *) cat <] : The name of the Git repository of YOUR fork at Github. Default: origin EOF exit 1 ;; esac ================================================ FILE: setup.cfg ================================================ [metadata] description-file = README.md ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # $Id: setup.py,v 1.1 2002/06/21 18:10:49 jgoerzen Exp $ # IMAP synchronization # Module: installer # COPYRIGHT # # Copyright (C) 2002 - 2018 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import os from distutils.core import setup, Command import offlineimap import logging from test.OLItest import TextTestRunner, TestLoader, OLITestLib class TestCommand(Command): """runs the OLI testsuite""" description = """Runs the test suite. In order to execute only a single test, you could also issue e.g. 'python -m unittest test.tests.test_01_basic.TestBasicFunctions.test_01_olistartup' on the command line.""" user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): logging.basicConfig(format='%(message)s') # set credentials and OfflineImap command to be executed: OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') suite = TestLoader().discover('./test/tests') #TODO: failfast does not seem to exist in python2.6? TextTestRunner(verbosity=2,failfast=True).run(suite) setup(name = "offlineimap", version = offlineimap.__version__, description = offlineimap.__description__, long_description = offlineimap.__description__, author = offlineimap.__author__, author_email = offlineimap.__author_email__, url = offlineimap.__homepage__, packages = ['offlineimap', 'offlineimap.folder', 'offlineimap.repository', 'offlineimap.ui', 'offlineimap.utils'], scripts = ['bin/offlineimap'], license = offlineimap.__copyright__ + \ ", Licensed under the GPL version 2", cmdclass = { 'test': TestCommand} ) ================================================ FILE: snapcraft.yaml ================================================ name: offlineimap version: git summary: OfflineIMAP description: | OfflineIMAP is software that downloads your email mailbox(es) as local Maildirs. OfflineIMAP will synchronize both sides via IMAP. grade: devel confinement: devmode apps: offlineimap: command: bin/offlineimap parts: offlineimap: plugin: python python-version: python2 source: . ================================================ FILE: test/.gitignore ================================================ credentials.conf tmp_* *.pyc OLItest/*.pyc tests/*.pyc ================================================ FILE: test/OLItest/TestRunner.py ================================================ # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import offlineimap.virtual_imaplib2 as imaplib import unittest import logging import os import re import sys import shutil import subprocess import tempfile import random random.seed() from offlineimap.CustomConfig import CustomConfigParser from . import default_conf class OLITestLib(): cred_file = None testdir = None """Absolute path of the current temporary test directory""" cmd = None """command that will be executed to invoke offlineimap""" def __init__(self, cred_file = None, cmd='offlineimap'): """ :param cred_file: file of the configuration snippet for authenticating against the test IMAP server(s). :param cmd: command that will be executed to invoke offlineimap""" OLITestLib.cred_file = cred_file if not os.path.isfile(cred_file): raise UserWarning("Please copy 'credentials.conf.sample' to '%s' " "and set your credentials there." % cred_file) OLITestLib.cmd = cmd @classmethod def create_test_dir(cls, suffix=''): """Creates a test directory and places OLI config there Note that this is a class method. There can only be one test directory at a time. OLITestLib is not suited for running several tests in parallel. The user is responsible for cleaning that up herself.""" assert cls.cred_file != None # creating temporary dir for testing in same dir as credentials.conf cls.testdir = os.path.abspath( tempfile.mkdtemp(prefix='tmp_%s_'%suffix, dir=os.path.dirname(cls.cred_file))) cls.write_config_file() return cls.testdir @classmethod def get_default_config(cls): """Creates a default ConfigParser file and returns it The returned config can be manipulated and then saved with write_config_file()""" #TODO, only do first time and cache then for subsequent calls? assert cls.cred_file != None assert cls.testdir != None config = CustomConfigParser() config.readfp(default_conf) default_conf.seek(0) # rewind config_file to start config.read(cls.cred_file) config.set("general", "metadata", cls.testdir) return config @classmethod def write_config_file(cls, config=None): """Creates a OLI configuration file It is created in testdir (so create_test_dir has to be called earlier) using the credentials information given (so they had to be set earlier). Failure to do either of them will raise an AssertionException. If config is None, a default one will be used via get_default_config, otherwise it needs to be a config object derived from that.""" if config is None: config = cls.get_default_config() localfolders = os.path.join(cls.testdir, 'mail') config.set("Repository Maildir", "localfolders", localfolders) with open(os.path.join(cls.testdir, 'offlineimap.conf'), "wt") as f: config.write(f) @classmethod def delete_test_dir(cls): """Deletes the current test directory The users is responsible for cleaning that up herself.""" if os.path.isdir(cls.testdir): shutil.rmtree(cls.testdir) @classmethod def run_OLI(cls): """Runs OfflineImap :returns: (rescode, stdout (as unicode)) """ try: output = subprocess.check_output( [cls.cmd, "-c%s" % os.path.join(cls.testdir, 'offlineimap.conf')], shell=False) except subprocess.CalledProcessError as e: return (e.returncode, e.output.decode('utf-8')) return (0, output.decode('utf-8')) @classmethod def delete_remote_testfolders(cls, reponame=None): """Delete all INBOX.OLITEST* folders on the remote IMAP repository reponame: All on `reponame` or all IMAP-type repositories if None""" config = cls.get_default_config() if reponame: sections = ['Repository {0}'.format(reponame)] else: sections = [r for r in config.sections() \ if r.startswith('Repository')] sections = [s for s in sections if config.get(s, 'Type').lower() == 'imap'] for sec in sections: # Connect to each IMAP repo and delete all folders # matching the folderfilter setting. We only allow basic # settings and no fancy password getting here... # 1) connect and get dir listing host = config.get(sec, 'remotehost') user = config.get(sec, 'remoteuser') passwd = config.get(sec, 'remotepass') imapobj = imaplib.IMAP4(host) imapobj.login(user, passwd) res_t, data = imapobj.list() assert res_t == 'OK' dirs = [] for d in data: if d == '': continue if isinstance(d, tuple): # literal (unquoted) folder = b'"%s"' % d[1].replace('"', '\\"') else: m = re.search(br''' [ ] # space (?P (?P"?) # starting quote ([^"]|\\")* # a non-quote or a backslashded quote (?P=quote))$ # ending quote ''', d, flags=re.VERBOSE) folder = bytearray(m.group('dir')) if not m.group('quote'): folder = '"%s"' % folder #folder = folder.replace(br'\"', b'"') # remove quoting dirs.append(folder) # 2) filter out those not starting with INBOX.OLItest and del... dirs = [d for d in dirs if d.startswith(b'"INBOX.OLItest') or d.startswith(b'"INBOX/OLItest')] for folder in dirs: res_t, data = imapobj.delete(folder) assert res_t == 'OK', "Folder deletion of {0} failed with error"\ ":\n{1} {2}".format(folder.decode('utf-8'), res_t, data) imapobj.logout() @classmethod def create_maildir(cls, folder): """Create empty maildir 'folder' in our test maildir Does not fail if it already exists""" assert cls.testdir != None maildir = os.path.join(cls.testdir, 'mail', folder) for subdir in ('','tmp','cur','new'): try: os.makedirs(os.path.join(maildir, subdir)) except OSError as e: if e.errno != 17: # 'already exists' is ok. raise @classmethod def delete_maildir(cls, folder): """Delete maildir 'folder' in our test maildir Does not fail if not existing""" assert cls.testdir != None maildir = os.path.join(cls.testdir, 'mail', folder) shutil.rmtree(maildir, ignore_errors=True) @classmethod def create_mail(cls, folder, mailfile=None, content=None): """Create a mail in maildir 'folder'/new Use default mailfilename if not given. Use some default content if not given""" assert cls.testdir != None while True: # Loop till we found a unique filename mailfile = '{0}:2,'.format(random.randint(0,999999999)) mailfilepath = os.path.join(cls.testdir, 'mail', folder, 'new', mailfile) if not os.path.isfile(mailfilepath): break with open(mailfilepath,"wb") as mailf: mailf.write(b'''From: test Subject: Boo Date: 1 Jan 1980 To: test@offlineimap.org Content here.''') @classmethod def count_maildir_mails(cls, folder): """Returns the number of mails in maildir 'folder' Counting only those in cur&new (ignoring tmp).""" assert cls.testdir != None maildir = os.path.join(cls.testdir, 'mail', folder) boxes, mails = 0, 0 for dirpath, dirs, files in os.walk(maildir, False): if set(dirs) == set(['cur', 'new', 'tmp']): # New maildir folder boxes += 1 #raise RuntimeError("%s is not Maildir" % maildir) if dirpath.endswith(('/cur', '/new')): mails += len(files) return boxes, mails # find UID in a maildir filename re_uidmatch = re.compile(',U=(\d+)') @classmethod def get_maildir_uids(cls, folder): """Returns a list of maildir mail uids, 'None' if no valid uid""" assert cls.testdir != None mailfilepath = os.path.join(cls.testdir, 'mail', folder) assert os.path.isdir(mailfilepath) ret = [] for dirpath, dirs, files in os.walk(mailfilepath): if not dirpath.endswith((os.path.sep + 'new', os.path.sep + 'cur')): continue # only /new /cur are interesting for file in files: m = cls.re_uidmatch.search(file) uid = m.group(1) if m else None ret.append(uid) return ret ================================================ FILE: test/OLItest/__init__.py ================================================ # OfflineImap test library # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA __all__ = ['OLITestLib', 'TextTestRunner','TestLoader'] __productname__ = 'OfflineIMAP Test suite' __version__ = '0' __copyright__ = "Copyright 2012- Sebastian Spaeth & contributors" __author__ = 'Sebastian Spaeth' __author_email__= 'Sebastian@SSpaeth.de' __description__ = 'Moo' __license__ = "Licensed under the GNU GPL v2+ (v2 or any later version)" __homepage__ = "http://www.offlineimap.org" banner = """%(__productname__)s %(__version__)s %(__license__)s""" % locals() import unittest from unittest import TestLoader, TextTestRunner from .globals import default_conf from .TestRunner import OLITestLib ================================================ FILE: test/OLItest/globals.py ================================================ #Constants, that don't rely on anything else in the module # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA try: from cStringIO import StringIO except ImportError: #python3 from io import StringIO default_conf=StringIO("""[general] #will be set automatically metadata = accounts = test ui = quiet [Account test] localrepository = Maildir remoterepository = IMAP [Repository Maildir] Type = Maildir # will be set automatically during tests localfolders = [Repository IMAP] type=IMAP # Don't hammer the server with too many connection attempts: maxconnections=1 folderfilter= lambda f: f.startswith('INBOX.OLItest') or f.startswith('INBOX/OLItest') """) ================================================ FILE: test/README ================================================ Documentation for the OfflineImap Test suite. How to run the tests ==================== - Copy the credentials.conf.sample to credentials.conf and insert credentials for an IMAP account and a Gmail account. Delete the Gmail section if you don't have a Gmail account. Do note, that the tests will change the account and upload/delete/modify it's contents and folder structure. So don't use a real used account here... - go to the top level dir (one above this one) and execute: 'python setup.py test' System requirements =================== This test suite depend on python>=2.7 to run out of the box. If you want to run this with python 2.6 you will need to install the backport from http://pypi.python.org/pypi/unittest2 instead. ================================================ FILE: test/__init__.py ================================================ ================================================ FILE: test/credentials.conf.sample ================================================ [Repository IMAP] type = IMAP remotehost = localhost ssl = no #sslcacertfile = #cert_fingerprint = remoteuser = user@domain remotepass = SeKr3t [Repository Gmail] type = Gmail remoteuser = user@domain remotepass = SeKr3t ================================================ FILE: test/tests/__init__.py ================================================ ================================================ FILE: test/tests/test_00_globals.py ================================================ #!/usr/bin/env python # Copyright 2013 Eygene A. Ryabinkin from offlineimap import globals import unittest class Opt: def __init__(self): self.one = "baz" self.two = 42 self.three = True class TestOfflineimapGlobals(unittest.TestCase): @classmethod def setUpClass(klass): klass.o = Opt() globals.set_options (klass.o) def test_initial_state(self): for k in self.o.__dict__.keys(): self.assertTrue(getattr(self.o, k) == getattr(globals.options, k)) def test_object_changes(self): self.o.one = "one" self.o.two = 119 self.o.three = False return self.test_initial_state() def test_modification(self): with self.assertRaises(AttributeError): globals.options.two = True def test_deletion(self): with self.assertRaises(RuntimeError): del globals.options.three def test_nonexistent_key(self): with self.assertRaises(AttributeError): a = globals.options.nosuchoption def test_double_init(self): with self.assertRaises(ValueError): globals.set_options (True) if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestOfflineimapGlobals) unittest.TextTestRunner(verbosity=2).run(suite) ================================================ FILE: test/tests/test_00_imaputil.py ================================================ # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import unittest import logging from offlineimap import imaputil from offlineimap.ui import UI_LIST, setglobalui from offlineimap.CustomConfig import CustomConfigParser from test.OLItest import OLITestLib # Things need to be setup first, usually setup.py initializes everything. # but if e.g. called from command line, we take care of default values here: if not OLITestLib.cred_file: OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') def setUpModule(): logging.info("Set Up test module %s" % __name__) tdir = OLITestLib.create_test_dir(suffix=__name__) def tearDownModule(): logging.info("Tear Down test module") # comment out next line to keep testdir after test runs. TODO: make nicer OLITestLib.delete_test_dir() #Stuff that can be used #self.assertEqual(self.seq, range(10)) # should raise an exception for an immutable sequence #self.assertRaises(TypeError, random.shuffle, (1,2,3)) #self.assertTrue(element in self.seq) #self.assertFalse(element in self.seq) class TestInternalFunctions(unittest.TestCase): """While the other test files test OfflineImap as a program, these tests directly invoke internal helper functions to guarantee that they deliver results as expected""" @classmethod def setUpClass(cls): #This is run before all tests in this class config= OLITestLib.get_default_config() setglobalui(UI_LIST['quiet'](config)) def test_01_imapsplit(self): """Test imaputil.imapsplit()""" res = imaputil.imapsplit(b'(\\HasNoChildren) "." "INBOX.Sent"') self.assertEqual(res, [b'(\\HasNoChildren)', b'"."', b'"INBOX.Sent"']) res = imaputil.imapsplit(b'"mo\\" o" sdfsdf') self.assertEqual(res, [b'"mo\\" o"', b'sdfsdf']) def test_02_flagsplit(self): """Test imaputil.flagsplit()""" res = imaputil.flagsplit(b'(\\Draft \\Deleted)') self.assertEqual(res, [b'\\Draft', b'\\Deleted']) res = imaputil.flagsplit(b'(FLAGS (\\Seen Old) UID 4807)') self.assertEqual(res, [b'FLAGS', b'(\\Seen Old)', b'UID', b'4807']) def test_04_flags2hash(self): """Test imaputil.flags2hash()""" res = imaputil.flags2hash(b'(FLAGS (\\Seen Old) UID 4807)') self.assertEqual(res, {b'FLAGS': b'(\\Seen Old)', b'UID': b'4807'}) def test_05_flagsimap2maildir(self): """Test imaputil.flagsimap2maildir()""" res = imaputil.flagsimap2maildir(b'(\\Draft \\Deleted)') self.assertEqual(res, set(b'DT')) def test_06_flagsmaildir2imap(self): """Test imaputil.flagsmaildir2imap()""" res = imaputil.flagsmaildir2imap(set(b'DR')) self.assertEqual(res, b'(\\Answered \\Draft)') # test all possible flags res = imaputil.flagsmaildir2imap(set(b'SRFTD')) self.assertEqual(res, b'(\\Answered \\Deleted \\Draft \\Flagged \\Seen)') def test_07_uid_sequence(self): """Test imaputil.uid_sequence()""" res = imaputil.uid_sequence([1,2,3,4,5,10,12,13]) self.assertEqual(res, b'1:5,10,12:13') ================================================ FILE: test/tests/test_01_basic.py ================================================ # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import random import unittest import logging import os, sys from test.OLItest import OLITestLib # Things need to be setup first, usually setup.py initializes everything. # but if e.g. called from command line, we take care of default values here: if not OLITestLib.cred_file: OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') def setUpModule(): logging.info("Set Up test module %s" % __name__) tdir = OLITestLib.create_test_dir(suffix=__name__) def tearDownModule(): logging.info("Tear Down test module") OLITestLib.delete_test_dir() #Stuff that can be used #self.assertEqual(self.seq, range(10)) # should raise an exception for an immutable sequence #self.assertRaises(TypeError, random.shuffle, (1,2,3)) #self.assertTrue(element in self.seq) #self.assertFalse(element in self.seq) class TestBasicFunctions(unittest.TestCase): def setUp(self): OLITestLib.delete_remote_testfolders() def tearDown(self): OLITestLib.delete_remote_testfolders() def test_01_olistartup(self): """Tests if OLI can be invoked without exceptions Cleans existing remote test folders. Then syncs all "OLItest* (specified in the default config) to our local Maildir. The result should be 0 folders and 0 mails.""" code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) def test_02_createdir(self): """Create local 'OLItest 1', sync""" OLITestLib.delete_maildir('') #Delete all local maildir folders OLITestLib.create_maildir('INBOX.OLItest 1') code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) def test_03_createdir_quote(self): """Create local 'OLItest "1"' maildir, sync Folder names with quotes used to fail and have been fixed, so one is included here as a small challenge.""" OLITestLib.delete_maildir('') #Delete all local maildir folders OLITestLib.create_maildir('INBOX.OLItest "1"') code, res = OLITestLib.run_OLI() if 'unallowed folder' in res: raise unittest.SkipTest("remote server doesn't handle quote") self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(1,0), msg="Expected 1 folders and 0 " "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) def test_04_nametransmismatch(self): """Create mismatching remote and local nametrans rules This should raise an error.""" config = OLITestLib.get_default_config() config.set('Repository IMAP', 'nametrans', 'lambda f: f' ) config.set('Repository Maildir', 'nametrans', 'lambda f: f + "moo"' ) OLITestLib.write_config_file(config) code, res = OLITestLib.run_OLI() #logging.warn("%s %s "% (code, res)) # We expect an INFINITE FOLDER CREATION WARNING HERE.... mismatch = "ERROR: INFINITE FOLDER CREATION DETECTED!" in res self.assertEqual(mismatch, True, msg="Mismatching nametrans rules did " "NOT trigger an 'infinite folder generation' error. Output was:\n" "{0}".format(res)) # Write out default config file again OLITestLib.write_config_file() def test_05_createmail(self): """Create mail in OLItest 1, sync, wipe folder sync Currently, this will mean the folder will be recreated locally. At some point when remote folder deletion is implemented, this behavior will change.""" OLITestLib.delete_remote_testfolders() OLITestLib.delete_maildir('') #Delete all local maildir folders OLITestLib.create_maildir('INBOX.OLItest') OLITestLib.create_mail('INBOX.OLItest') code, res = OLITestLib.run_OLI() #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(1,1), msg="Expected 1 folders and 1 " "mails, but sync led to {0} folders and {1} mails".format( boxes, mails)) # The local Mail should have been assigned a proper UID now, check! uids = OLITestLib.get_maildir_uids('INBOX.OLItest') self.assertFalse (None in uids, msg = "All mails should have been "+ \ "assigned the IMAP's UID number, but {0} messages had no valid ID "\ .format(len([None for x in uids if x==None]))) def test_06_createfolders(self): """Test if createfolders works as expected Create a local Maildir, then sync with remote "createfolders" disabled. Delete local Maildir and sync. We should have no new local maildir then. TODO: Rewrite this test to directly test and count the remote folders when the helper functions have been written""" config = OLITestLib.get_default_config() config.set('Repository IMAP', 'createfolders', 'False' ) OLITestLib.write_config_file(config) # delete all remote and local testfolders OLITestLib.delete_remote_testfolders() OLITestLib.delete_maildir('') OLITestLib.create_maildir('INBOX.OLItest') code, res = OLITestLib.run_OLI() #logging.warn("%s %s "% (code, res)) self.assertEqual(res, "") OLITestLib.delete_maildir('INBOX.OLItest') code, res = OLITestLib.run_OLI() self.assertEqual(res, "") boxes, mails = OLITestLib.count_maildir_mails('') self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0 " "mails, but sync led to {} folders and {} mails".format( boxes, mails)) ================================================ FILE: test/tests/test_02_MappedIMAP.py ================================================ # Copyright (C) 2012- Sebastian Spaeth & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import random import unittest import logging import os, sys from test.OLItest import OLITestLib # Things need to be setup first, usually setup.py initializes everything. # but if e.g. called from command line, we take care of default values here: if not OLITestLib.cred_file: OLITestLib(cred_file='./test/credentials.conf', cmd='./offlineimap.py') def setUpModule(): logging.info("Set Up test module %s" % __name__) tdir = OLITestLib.create_test_dir(suffix=__name__) def tearDownModule(): logging.info("Tear Down test module") OLITestLib.delete_test_dir() #Stuff that can be used #self.assertEqual(self.seq, range(10)) # should raise an exception for an immutable sequence #self.assertRaises(TypeError, random.shuffle, (1,2,3)) #self.assertTrue(element in self.seq) #self.assertFalse(element in self.seq) class TestBasicFunctions(unittest.TestCase): #@classmethod #def setUpClass(cls): #This is run before all tests in this class # cls._connection = createExpensiveConnectionObject() #@classmethod #This is run after all tests in this class #def tearDownClass(cls): # cls._connection.destroy() # This will be run before each test #def setUp(self): # self.seq = range(10) def test_01_MappedImap(self): """Tests if a MappedIMAP sync can be invoked without exceptions Cleans existing remote test folders. Then syncs all "OLItest* (specified in the default config) to our local IMAP (Gmail). The result should be 0 folders and 0 mails.""" pass #TODO #OLITestLib.delete_remote_testfolders() #code, res = OLITestLib.run_OLI() #self.assertEqual(res, "") #boxes, mails = OLITestLib.count_maildir_mails('') #self.assertTrue((boxes, mails)==(0,0), msg="Expected 0 folders and 0" # "mails, but sync led to {} folders and {} mails".format( # boxes, mails)) ================================================ FILE: tests/.gitignore ================================================ *.pyc ================================================ FILE: tests/create_conf_file.py ================================================ #!/bin/env python # Copyright 2018 Espace LLC/espacenetworks.com. Written by @chris001. # This must be run from the main directory of the offlineimap project. # Typically this script will be run by Travis to create the config files needed for running the automated tests. # python ./tests/create_conf_file.py # Input: Seven shell environment variables. # Output: it writes the config settings to "filename" (./oli-travis.conf) and "additionalfilename" (./test/credentials.conf). # "filename" is used by normal run of ./offlineimap -c ./oli-travis.conf , "additionalfilename" is used by "pytest". # They are the same conf file, copie to two different locations for convenience. import os import shutil try: import ConfigParser Config = ConfigParser.ConfigParser() except ImportError: import configparser Config = configparser.ConfigParser() filename = "./oli-travis.conf" additionalfilename = "./test/credentials.conf" # for the 'pytest' which automatically finds and runs the unittests. #TODO: detect OS we running on now, and set sslcacertfile location accordingly. sslcacertfile = "/etc/pki/tls/cert.pem" # CentOS 7 sslcacertfile = "" # TODO: https://gist.github.com/1stvamp/2158128 Current Mac OSX now must download the cacertfile. sslcacertfile = "/etc/ssl/certs/ca-certificates.crt" # Ubuntu Trusty 14.04 (Travis linux test container 2018.) if os.environ["TRAVIS_OS_NAME"] == "osx": sslcacertfile = os.environ["OSX_BREW_SSLCACERTFILE"] # lets create that config file. cfgfile = open(filename,'w') # add the settings to the structure of the file, and lets write it out. sect = 'general' Config.add_section(sect) Config.set(sect,'accounts','Test') Config.set(sect,'maxsyncaccounts', '1') sect = 'Account Test' Config.add_section(sect) Config.set(sect,'localrepository','IMAP') # Outlook. Config.set(sect,'remoterepository', 'Gmail') ### "Repository IMAP" is hardcoded in test/OLItest/TestRunner.py it should dynamically get the Repository names but it doesn't. sect = 'Repository IMAP' # Outlook. Config.add_section(sect) Config.set(sect,'type','IMAP') Config.set(sect,'remotehost', 'imap-mail.outlook.com') Config.set(sect,'remoteport', '993') Config.set(sect,'auth_mechanisms', os.environ["OUTLOOK_AUTH"]) Config.set(sect,'ssl', 'True') #Config.set(sect,'tls_level', 'tls_compat') #Default is 'tls_compat'. #Config.set(sect,'ssl_version', 'tls1_2') # Leave this unset. Will auto select between tls1_1 and tls1_2 for tls_secure. Config.set(sect,'sslcacertfile', sslcacertfile) Config.set(sect,'remoteuser', os.environ["secure_outlook_email_address"]) Config.set(sect,'remotepass', os.environ["secure_outlook_email_pw"]) Config.set(sect,'createfolders', 'True') Config.set(sect,'folderfilter', 'lambda f: f not in ["Inbox", "[Gmail]/All Mail"]') #Capitalization of Inbox INBOX was causing runtime failure. #Config.set(sect,'folderfilter', 'lambda f: f not in ["[Gmail]/All Mail"]') ### "Repository Gmail" is also hardcoded into test/OLItest/TestRunner.py sect = 'Repository Gmail' Config.add_section(sect) Config.set(sect,'type', 'Gmail') Config.set(sect,'remoteport', '993') Config.set(sect,'auth_mechanisms', os.environ["GMAIL_AUTH"]) Config.set(sect,'oauth2_client_id', os.environ["secure_gmail_oauth2_client_id"]) Config.set(sect,'oauth2_client_secret', os.environ["secure_gmail_oauth2_client_secret"]) Config.set(sect,'oauth2_refresh_token', os.environ["secure_gmail_oauth2_refresh_token"]) Config.set(sect,'remoteuser', os.environ["secure_gmail_email_address"]) Config.set(sect,'ssl', 'True') #Config.set(sect,'tls_level', 'tls_compat') #Config.set(sect,'ssl_version', 'tls1_2') Config.set(sect,'sslcacertfile', sslcacertfile) Config.set(sect,'createfolders', 'True') Config.set(sect,'folderfilter', 'lambda f: f not in ["INBOX", "[Gmail]/All Mail"]') Config.write(cfgfile) cfgfile.close() shutil.copy(filename, additionalfilename) ================================================ FILE: tests/requirements.txt ================================================ pytest pytest-cov coverage codecov